This is an automated email from the ASF dual-hosted git repository. bzp2010 pushed a commit to branch bzp/feat-standalone-isolate-bad-resources in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
commit e1627014cdb42ea1ca9e6c9a8b7465e24e77edaf Author: bzp2010 <[email protected]> AuthorDate: Wed Sep 16 13:04:16 2026 +0800 e2e --- test/e2e/crds/v1alpha1/consumer.go | 124 +++++++++++ test/e2e/crds/v2/isolation.go | 435 +++++++++++++++++++++++++++++++++++++ test/e2e/crds/v2/status.go | 190 ---------------- test/e2e/crds/v2/streamroute.go | 83 +++++++ test/e2e/gatewayapi/gateway.go | 93 ++++++++ test/e2e/ingress/ingress.go | 112 ++++++++++ 6 files changed, 847 insertions(+), 190 deletions(-) diff --git a/test/e2e/crds/v1alpha1/consumer.go b/test/e2e/crds/v1alpha1/consumer.go index 3dedd0a8..56bb1051 100644 --- a/test/e2e/crds/v1alpha1/consumer.go +++ b/test/e2e/crds/v1alpha1/consumer.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + gomegatypes "github.com/onsi/gomega/types" "k8s.io/apimachinery/pkg/types" "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" @@ -621,4 +622,127 @@ spec: }) }) }) + + // A Consumer the data plane rejects must not stop the other Consumers under the same + // GatewayProxy from being applied, and one rejected credential must not take the + // Consumer's other credentials with it. What makes them rejected is checked by every + // backend: a known plugin configured in a way its own check_schema refuses, and a + // credential config of the wrong type. + Context("Bad resource isolation", func() { + var consumerWithPlugin = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: Consumer +metadata: + name: consumer-rejected +spec: + gatewayRef: + name: %s + credentials: + - type: key-auth + name: key-auth-sample + config: + key: rejected-key + plugins: + - name: limit-count + config: + count: %d + time_window: 60 + rejected_code: 503 + key: remote_addr +` + var validConsumer = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: Consumer +metadata: + name: consumer-valid +spec: + gatewayRef: + name: %s + credentials: + - type: key-auth + name: key-auth-sample + config: + key: valid-key +` + var consumerWithCredentials = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: Consumer +metadata: + name: consumer-mixed +spec: + gatewayRef: + name: %s + credentials: + - type: key-auth + name: valid-credential + config: + key: mixed-key + - type: key-auth + name: rejected-credential + config: + key: %s +` + authenticates := func(key string, status int) { + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{"apikey": key}, + Check: scaffold.WithExpectedStatus(status), + }) + } + consumerStatus := func(name string, matchers ...gomegatypes.GomegaMatcher) { + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString("consumer", name, "-o", "yaml", "-n", s.Namespace()) + return output + }).Should(And(matchers...)) + } + + It("isolates a rejected Consumer", func() { + By("apply a valid and a rejected Consumer") + err = s.CreateResourceFromString(fmt.Sprintf(consumerWithPlugin, s.Namespace(), 0)) + Expect(err).NotTo(HaveOccurred(), "creating the rejected Consumer") + err = s.CreateResourceFromString(fmt.Sprintf(validConsumer, s.Namespace())) + Expect(err).NotTo(HaveOccurred(), "creating the valid Consumer") + + By("the valid Consumer authenticates, the rejected one does not") + authenticates("valid-key", 200) + authenticates("rejected-key", 401) + consumerStatus("consumer-rejected", + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ) + + By("fix the rejected Consumer") + err = s.CreateResourceFromString(fmt.Sprintf(consumerWithPlugin, s.Namespace(), 100)) + Expect(err).NotTo(HaveOccurred(), "updating the Consumer") + + By("both Consumers authenticate") + authenticates("valid-key", 200) + authenticates("rejected-key", 200) + }) + + It("isolates a rejected credential without taking the Consumer's other credentials", func() { + By("apply a Consumer with one valid and one rejected credential") + // key has to be a string, so the data plane refuses this credential. + err = s.CreateResourceFromString(fmt.Sprintf(consumerWithCredentials, s.Namespace(), "123")) + Expect(err).NotTo(HaveOccurred(), "creating the Consumer") + + By("the valid credential authenticates and the Consumer reports the dropped one") + authenticates("mixed-key", 200) + consumerStatus("consumer-mixed", + ContainSubstring(`type: PartiallyInvalid`), + ContainSubstring(`rejected-credential`), + ) + + By("fix the rejected credential") + err = s.CreateResourceFromString(fmt.Sprintf(consumerWithCredentials, s.Namespace(), `"fixed-key"`)) + Expect(err).NotTo(HaveOccurred(), "updating the Consumer") + + By("both credentials authenticate") + authenticates("mixed-key", 200) + authenticates("fixed-key", 200) + consumerStatus("consumer-mixed", Not(ContainSubstring(`type: PartiallyInvalid`))) + }) + }) }) diff --git a/test/e2e/crds/v2/isolation.go b/test/e2e/crds/v2/isolation.go new file mode 100644 index 00000000..a4be157c --- /dev/null +++ b/test/e2e/crds/v2/isolation.go @@ -0,0 +1,435 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package v2 + +import ( + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + gomegatypes "github.com/onsi/gomega/types" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +// Every case applies a valid and a rejected resource together, so the first sync carries +// both: the sync converges when the rejected one is excluded and the valid one is served. +// Fixing the rejected one then has to bring it in too. +// +// What makes a resource rejected is always something the data plane checks for every +// backend: a value outside the schema's range, or a known plugin configured in a way its +// own check_schema refuses. An unknown plugin name is not used, since apisix-standalone +// accepts those. +var _ = Describe("Test bad resource isolation", Label("apisix.apache.org", "v2", "isolation"), func() { + s := scaffold.NewDefaultScaffold() + + // rejectedPlugin is a plugin every backend loads, configured with a count the plugin + // itself refuses (it has to be greater than 0). + const rejectedPlugin = ` + plugins: + - name: limit-count + enable: true + config: + count: 0 + time_window: 60 + rejected_code: 503 + key: remote_addr +` + const acceptedPlugin = ` + plugins: + - name: limit-count + enable: true + config: + count: 100 + time_window: 60 + rejected_code: 503 + key: remote_addr +` + // routes is one ApisixRoute serving valid.example.com and one serving + // rejected.example.com, whose plugin configuration is filled in per case. + const routes = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: valid + namespace: %s +spec: + ingressClassName: %s + http: + - name: rule0 + match: + hosts: + - valid.example.com + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +--- +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: rejected + namespace: %s +spec: + ingressClassName: %s + http: + - name: rule0 + match: + hosts: + - rejected.example.com + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +%s +` + + expectServed := func(host string) { + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: host, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + } + expectStatus := func(resource, name string, matchers ...gomegatypes.GomegaMatcher) { + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString(resource, name, "-o", "yaml", "-n", s.Namespace()) + return output + }).Should(And(matchers...)) + } + apply := func(yaml string) { + Expect(s.CreateResourceFromString(yaml)).NotTo(HaveOccurred(), "applying resources") + } + + BeforeEach(func() { + By("create GatewayProxy") + Expect(s.CreateResourceFromString(s.GetGatewayProxySpec())).NotTo(HaveOccurred(), "creating GatewayProxy") + + By("create IngressClass") + Expect(s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "")).NotTo(HaveOccurred(), "creating IngressClass") + }) + + It("isolates a rejected route", func() { + By("apply a valid and a rejected ApisixRoute") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), rejectedPlugin)) + + By("the valid ApisixRoute is served and the rejected one reports why it is not") + expectServed("valid.example.com") + expectStatus("ar", "rejected", + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ContainSubstring(`failed to check the configuration of plugin limit-count`), + ) + + By("fix the rejected ApisixRoute") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), acceptedPlugin)) + + By("both ApisixRoutes are served") + expectServed("valid.example.com") + expectServed("rejected.example.com") + expectStatus("ar", "rejected", ContainSubstring(`reason: Accepted`)) + }) + + It("isolates a rejected rule of an otherwise valid route", func() { + const partialRoute = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: partial + namespace: %s +spec: + ingressClassName: %s + http: + - name: served + match: + hosts: + - valid.example.com + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + - name: rejected + match: + hosts: + - rejected.example.com + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +%s +` + By("apply an ApisixRoute with one valid and one rejected rule") + apply(fmt.Sprintf(partialRoute, s.Namespace(), s.Namespace(), rejectedPlugin)) + + By("the valid rule is served and the ApisixRoute reports the dropped one") + expectServed("valid.example.com") + expectStatus("ar", "partial", + ContainSubstring(`type: PartiallyInvalid`), + ContainSubstring(`failed to check the configuration of plugin limit-count`), + ) + + By("fix the rejected rule") + apply(fmt.Sprintf(partialRoute, s.Namespace(), s.Namespace(), acceptedPlugin)) + + By("both rules are served") + expectServed("valid.example.com") + expectServed("rejected.example.com") + expectStatus("ar", "partial", Not(ContainSubstring(`type: PartiallyInvalid`))) + }) + + It("isolates a route whose upstream configuration is rejected", func() { + // retries has no lower bound in the CRD and the data plane requires it to be at + // least 0, so this reaches the data plane and is rejected on schema grounds. + const upstream = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-service-e2e-test + namespace: %s +spec: + ingressClassName: %s + retries: %d +` + By("apply a valid and a rejected ApisixRoute, the rejected one using a rejected upstream configuration") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), "")) + apply(fmt.Sprintf(upstream, s.Namespace(), s.Namespace(), -1)) + + By("the rejected ApisixRoute reports why it is not served") + expectStatus("ar", "rejected", + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ) + + By("fix the upstream configuration") + apply(fmt.Sprintf(upstream, s.Namespace(), s.Namespace(), 1)) + + By("both ApisixRoutes are served") + expectServed("valid.example.com") + expectServed("rejected.example.com") + }) + + It("isolates the rule of a rejected named upstream", func() { + // A named upstream is referenced by its service's traffic-split, so dropping it + // alone would leave that reference dangling: the whole rule goes instead. + const namedUpstream = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: external + namespace: %s +spec: + ingressClassName: %s + retries: %d + externalNodes: + - type: Service + name: httpbin-service-e2e-test +--- +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: named-upstream + namespace: %s +spec: + ingressClassName: %s + http: + - name: served + match: + hosts: + - valid.example.com + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + - name: rejected + match: + hosts: + - rejected.example.com + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 + upstreams: + - name: external +` + By("apply an ApisixRoute whose second rule references a rejected ApisixUpstream") + apply(fmt.Sprintf(namedUpstream, s.Namespace(), s.Namespace(), -1, s.Namespace(), s.Namespace())) + + By("the valid rule is served and the ApisixRoute reports the dropped one") + expectServed("valid.example.com") + expectStatus("ar", "named-upstream", ContainSubstring(`type: PartiallyInvalid`)) + + By("fix the ApisixUpstream") + apply(fmt.Sprintf(namedUpstream, s.Namespace(), s.Namespace(), 1, s.Namespace(), s.Namespace())) + + By("both rules are served") + expectServed("valid.example.com") + expectServed("rejected.example.com") + }) + + It("isolates a rejected certificate", func() { + const tls = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixTls +metadata: + name: rejected + namespace: %s +spec: + ingressClassName: %s + hosts: + - ssl.example.com + secret: + name: rejected-cert + namespace: %s +` + cert, key := s.GenerateCert(GinkgoT(), []string{"ssl.example.com"}) + By("apply a valid ApisixRoute and an ApisixTls whose private key the data plane cannot parse") + // The certificate itself is valid, so the controller accepts it; only the data + // plane rejects the key. + Expect(s.NewKubeTlsSecret("rejected-cert", cert.String(), "-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----")). + NotTo(HaveOccurred(), "creating Secret") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), "")) + apply(fmt.Sprintf(tls, s.Namespace(), s.Namespace(), s.Namespace())) + + By("the ApisixRoutes are served and the ApisixTls reports why it is not") + expectServed("valid.example.com") + expectServed("rejected.example.com") + expectStatus("apisixtls", "rejected", + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ) + + By("fix the private key") + Expect(s.NewKubeTlsSecret("rejected-cert", cert.String(), key.String())).NotTo(HaveOccurred(), "updating Secret") + + By("the ApisixTls is accepted") + expectStatus("apisixtls", "rejected", ContainSubstring(`reason: Accepted`)) + }) + + It("isolates a rejected global rule", func() { + const globalRule = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixGlobalRule +metadata: + name: rejected + namespace: %s +spec: + ingressClassName: %s + plugins: + - name: limit-count + enable: true + config: + count: %d + time_window: 60 + rejected_code: 503 + key: remote_addr +` + By("apply a valid ApisixRoute and a rejected ApisixGlobalRule") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), "")) + apply(fmt.Sprintf(globalRule, s.Namespace(), s.Namespace(), 0)) + + By("the ApisixRoutes are served and the ApisixGlobalRule reports why it is not") + expectServed("valid.example.com") + expectServed("rejected.example.com") + expectStatus("apisixglobalrule", "rejected", + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ) + + By("fix the ApisixGlobalRule") + apply(fmt.Sprintf(globalRule, s.Namespace(), s.Namespace(), 100)) + + By("the ApisixGlobalRule is accepted and the routes keep being served") + expectStatus("apisixglobalrule", "rejected", ContainSubstring(`reason: Accepted`)) + expectServed("valid.example.com") + }) + + It("isolates a rejected GatewayProxy plugin", func() { + gatewayProxyWithPlugin := func(count int) string { + return s.GetGatewayProxySpec() + fmt.Sprintf(` plugins: + - name: limit-count + enabled: true + config: + count: %d + time_window: 60 + rejected_code: 503 + key: remote_addr +`, count) + } + By("apply a valid ApisixRoute and a GatewayProxy carrying a rejected plugin") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), "")) + apply(gatewayProxyWithPlugin(0)) + + By("the ApisixRoutes are served and the GatewayProxy reports the dropped plugin") + expectServed("valid.example.com") + expectServed("rejected.example.com") + expectStatus("gatewayproxy", "apisix-proxy-config", + ContainSubstring(`type: PluginsProgrammed`), + ContainSubstring(`reason: Invalid`), + ContainSubstring(`limit-count`), + ) + + By("fix the GatewayProxy plugin") + apply(gatewayProxyWithPlugin(100)) + + By("the GatewayProxy reports its plugins programmed") + expectStatus("gatewayproxy", "apisix-proxy-config", + ContainSubstring(`type: PluginsProgrammed`), + ContainSubstring(`reason: Programmed`), + ) + expectServed("valid.example.com") + }) + + It("isolates rejected GatewayProxy plugin metadata", func() { + gatewayProxyWithMetadata := func(logFormat string) string { + return s.GetGatewayProxySpec() + fmt.Sprintf(` pluginMetadata: + http-logger: + log_format: %s +`, logFormat) + } + By("apply a valid ApisixRoute and a GatewayProxy carrying rejected plugin metadata") + apply(fmt.Sprintf(routes, s.Namespace(), s.Namespace(), s.Namespace(), s.Namespace(), "")) + apply(gatewayProxyWithMetadata(`"not an object"`)) + + By("the ApisixRoutes are served and the GatewayProxy reports the dropped metadata") + expectServed("valid.example.com") + expectServed("rejected.example.com") + expectStatus("gatewayproxy", "apisix-proxy-config", + ContainSubstring(`type: PluginsProgrammed`), + ContainSubstring(`reason: Invalid`), + ContainSubstring(`http-logger`), + ) + + By("fix the plugin metadata") + apply(gatewayProxyWithMetadata(`{"host": "$host"}`)) + + By("the GatewayProxy reports its plugins programmed") + expectStatus("gatewayproxy", "apisix-proxy-config", + ContainSubstring(`type: PluginsProgrammed`), + ContainSubstring(`reason: Programmed`), + ) + expectServed("valid.example.com") + }) +}) diff --git a/test/e2e/crds/v2/status.go b/test/e2e/crds/v2/status.go index c832b8fa..d43ddde7 100644 --- a/test/e2e/crds/v2/status.go +++ b/test/e2e/crds/v2/status.go @@ -140,196 +140,6 @@ spec: }) }) - It("a rejected ApisixRoute does not block other resources and is retried once fixed", func() { - if os.Getenv("PROVIDER_TYPE") == framework.ProviderTypeAPISIXStandalone { - Skip("apisix standalone does not validate unknown plugins") - } - const routeYaml = ` -apiVersion: apisix.apache.org/v2 -kind: ApisixRoute -metadata: - name: %s - namespace: %s -spec: - ingressClassName: %s - http: - - name: rule0 - match: - hosts: - - %s - paths: - - /* - backends: - - serviceName: httpbin-service-e2e-test - servicePort: 80 - plugins: - - name: %s - enable: true -` - By("apply a rejected and a valid ApisixRoute") - err := s.CreateResourceFromString(fmt.Sprintf(routeYaml, "bad", s.Namespace(), s.Namespace(), "bad.example.com", "non-existent-plugin")) - Expect(err).NotTo(HaveOccurred(), "creating the rejected ApisixRoute") - err = s.CreateResourceFromString(fmt.Sprintf(routeYaml, "good", s.Namespace(), s.Namespace(), "good.example.com", "cors")) - Expect(err).NotTo(HaveOccurred(), "creating the valid ApisixRoute") - - By("the rejected ApisixRoute reports why it is not served") - s.RetryAssertion(func() string { - output, _ := s.GetOutputFromString("ar", "bad", "-o", "yaml", "-n", s.Namespace()) - return output - }).Should(And( - ContainSubstring(`status: "False"`), - ContainSubstring(`reason: SyncFailed`), - ContainSubstring(`unknown plugin [non-existent-plugin]`), - )) - - By("the valid ApisixRoute is served regardless") - s.RequestAssert(&scaffold.RequestAssert{ - Method: "GET", - Path: "/get", - Host: "good.example.com", - Check: scaffold.WithExpectedStatus(200), - }) - - By("fix the rejected ApisixRoute") - applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "bad"}, &apiv2.ApisixRoute{}, - fmt.Sprintf(routeYaml, "bad", s.Namespace(), s.Namespace(), "bad.example.com", "cors")) - - By("the fixed ApisixRoute is served") - s.RequestAssert(&scaffold.RequestAssert{ - Method: "GET", - Path: "/get", - Host: "bad.example.com", - Check: scaffold.WithExpectedStatus(200), - }) - }) - - It("a rejected rule is dropped while the rest of the ApisixRoute is served", func() { - if os.Getenv("PROVIDER_TYPE") == framework.ProviderTypeAPISIXStandalone { - Skip("apisix standalone does not validate unknown plugins") - } - const partialRouteYaml = ` -apiVersion: apisix.apache.org/v2 -kind: ApisixRoute -metadata: - name: partial - namespace: %s -spec: - ingressClassName: %s - http: - - name: valid - match: - hosts: - - valid.example.com - paths: - - /* - backends: - - serviceName: httpbin-service-e2e-test - servicePort: 80 - - name: rejected - match: - hosts: - - rejected.example.com - paths: - - /* - backends: - - serviceName: httpbin-service-e2e-test - servicePort: 80 - plugins: - - name: non-existent-plugin - enable: true -` - err := s.CreateResourceFromString(fmt.Sprintf(partialRouteYaml, s.Namespace(), s.Namespace())) - Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute") - - By("the ApisixRoute reports the dropped rule") - s.RetryAssertion(func() string { - output, _ := s.GetOutputFromString("ar", "partial", "-o", "yaml", "-n", s.Namespace()) - return output - }).Should(And( - ContainSubstring(`type: PartiallyInvalid`), - ContainSubstring(`unknown plugin [non-existent-plugin]`), - )) - - By("the valid rule is served") - s.RequestAssert(&scaffold.RequestAssert{ - Method: "GET", - Path: "/get", - Host: "valid.example.com", - Check: scaffold.WithExpectedStatus(200), - }) - }) - - It("a rejected ApisixGlobalRule does not block routes", func() { - if os.Getenv("PROVIDER_TYPE") == framework.ProviderTypeAPISIXStandalone { - Skip("apisix standalone does not validate unknown plugins") - } - const globalRuleYaml = ` -apiVersion: apisix.apache.org/v2 -kind: ApisixGlobalRule -metadata: - name: rejected - namespace: %s -spec: - ingressClassName: %s - plugins: - - name: non-existent-plugin - enable: true -` - err := s.CreateResourceFromString(fmt.Sprintf(globalRuleYaml, s.Namespace(), s.Namespace())) - Expect(err).NotTo(HaveOccurred(), "creating ApisixGlobalRule") - err = s.CreateResourceFromString(fmt.Sprintf(ar, s.Namespace(), s.Namespace())) - Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute") - - By("the ApisixGlobalRule reports why it is not served") - s.RetryAssertion(func() string { - output, _ := s.GetOutputFromString("apisixglobalrule", "rejected", "-o", "yaml", "-n", s.Namespace()) - return output - }).Should(And( - ContainSubstring(`status: "False"`), - ContainSubstring(`reason: SyncFailed`), - )) - - By("the ApisixRoute is served regardless") - s.RequestAssert(&scaffold.RequestAssert{ - Method: "GET", - Path: "/get", - Host: "httpbin", - Check: scaffold.WithExpectedStatus(200), - }) - }) - - It("a rejected GatewayProxy plugin is reported on the GatewayProxy and does not block routes", func() { - if os.Getenv("PROVIDER_TYPE") == framework.ProviderTypeAPISIXStandalone { - Skip("apisix standalone does not validate unknown plugins") - } - By("add a rejected plugin to the GatewayProxy") - err := s.CreateResourceFromString(s.GetGatewayProxySpec() + ` plugins: - - name: non-existent-plugin - enabled: true -`) - Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy") - err = s.CreateResourceFromString(fmt.Sprintf(ar, s.Namespace(), s.Namespace())) - Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute") - - By("the GatewayProxy reports the rejected plugin") - s.RetryAssertion(func() string { - output, _ := s.GetOutputFromString("gatewayproxy", "apisix-proxy-config", "-o", "yaml", "-n", s.Namespace()) - return output - }).Should(And( - ContainSubstring(`type: PluginsProgrammed`), - ContainSubstring(`reason: Invalid`), - ContainSubstring(`unknown plugin [non-existent-plugin]`), - )) - - By("the ApisixRoute is served regardless") - s.RequestAssert(&scaffold.RequestAssert{ - Method: "GET", - Path: "/get", - Host: "httpbin", - Check: scaffold.WithExpectedStatus(200), - }) - }) - It("dataplane unavailable", func() { By("apply ApisixRoute") arYaml := fmt.Sprintf(ar, s.Namespace(), s.Namespace()) diff --git a/test/e2e/crds/v2/streamroute.go b/test/e2e/crds/v2/streamroute.go index b27455b0..d3daa0f7 100644 --- a/test/e2e/crds/v2/streamroute.go +++ b/test/e2e/crds/v2/streamroute.go @@ -104,6 +104,89 @@ spec: }) }) + // A stream route the data plane rejects must not stop the other stream routes under + // the same GatewayProxy from being applied. limit-conn requires conn to be greater + // than 0, which every backend checks. + Context("Bad resource isolation", func() { + streamRoutes := ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: valid-tcp-route +spec: + ingressClassName: %s + stream: + - name: rule1 + protocol: TCP + match: + ingressPort: 9100 + backend: + serviceName: httpbin-service-e2e-test + servicePort: 80 +--- +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: rejected-tcp-route +spec: + ingressClassName: %s + stream: + - name: rule1 + protocol: TCP + match: + ingressPort: 9110 + backend: + serviceName: httpbin-service-e2e-test + servicePort: 80 + plugins: + - name: limit-conn + enable: true + config: + conn: %d + burst: 1 + default_conn_delay: 1 + key: remote_addr +` + It("isolates a rejected stream route", func() { + By("apply a valid and a rejected stream route") + err := s.CreateResourceFromString(fmt.Sprintf(streamRoutes, s.Namespace(), s.Namespace(), 0)) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes") + + By("the valid stream route proxies") + s.RequestAssert(&scaffold.RequestAssert{ + Client: s.NewAPISIXClientWithTCPProxy(), + Method: "GET", + Path: "/ip", + Check: scaffold.WithExpectedStatus(200), + }) + + By("the rejected ApisixRoute reports why it is not served") + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString("ar", "rejected-tcp-route", "-o", "yaml", "-n", s.Namespace()) + return output + }).Should(And( + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + )) + + By("fix the rejected stream route") + err = s.CreateResourceFromString(fmt.Sprintf(streamRoutes, s.Namespace(), s.Namespace(), 1)) + Expect(err).NotTo(HaveOccurred(), "updating ApisixRoutes") + + By("both stream routes are accepted") + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString("ar", "rejected-tcp-route", "-o", "yaml", "-n", s.Namespace()) + return output + }).Should(ContainSubstring(`reason: Accepted`)) + s.RequestAssert(&scaffold.RequestAssert{ + Client: s.NewAPISIXClientWithTCPProxy(), + Method: "GET", + Path: "/ip", + Check: scaffold.WithExpectedStatus(200), + }) + }) + }) + Context("TCP Proxy with TLS upstream", func() { apisixUpstream := ` apiVersion: apisix.apache.org/v2 diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index 5d169704..a80f60f1 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -231,6 +231,99 @@ spec: }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) }) + // A certificate the data plane rejects is reported on the listener that carries + // it, and does not stop the Gateway's other listeners from being programmed. + It("Check a certificate the data plane rejects is reported on its listener", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + + By("create one valid and one unparsable certificate") + createSecret(s, _secretName) + // The certificate itself is valid, so the controller accepts the reference; + // the private key is well-formed PEM the data plane cannot parse. + err = s.NewKubeTlsSecret("rejected-cert", Cert, "-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----") + Expect(err).NotTo(HaveOccurred(), "creating Secret") + + gatewayClassName := s.Namespace() + By("create GatewayClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(` +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: %s +spec: + controllerName: "%s" +`, gatewayClassName, s.GetControllerName()), "") + Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") + + By("create a Gateway whose second listener carries the rejected certificate") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %s +spec: + gatewayClassName: %s + listeners: + - name: accepted + protocol: HTTPS + port: 443 + hostname: %s + tls: + certificateRefs: + - kind: Secret + group: "" + name: %s + - name: rejected + protocol: HTTPS + port: 8443 + hostname: rejected.example.com + tls: + certificateRefs: + - kind: Secret + group: "" + name: rejected-cert + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +`, s.Namespace(), gatewayClassName, _hostAPI6, _secretName), s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating Gateway") + + By("the valid certificate is programmed") + Eventually(func(g Gomega) { + tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) + g.Expect(err).NotTo(HaveOccurred(), "list tls error") + g.Expect(tls).To(HaveLen(1), "only the accepted listener's certificate is programmed") + g.Expect(tls[0].Snis).To(ConsistOf(_hostAPI6)) + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) + + By("the rejected certificate is reported on its own listener") + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("Gateway", s.Namespace()) + }).Should(And( + ContainSubstring("name: rejected"), + ContainSubstring("reason: InvalidCertificateRef"), + ), "checking listener condition") + + By("fix the rejected certificate") + err = s.NewKubeTlsSecret("rejected-cert", Cert, Key) + Expect(err).NotTo(HaveOccurred(), "updating Secret") + + By("both certificates are programmed and no listener reports a rejection") + Eventually(func(g Gomega) { + tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) + g.Expect(err).NotTo(HaveOccurred(), "list tls error") + g.Expect(tls).To(HaveLen(2), "both listeners' certificates are programmed") + }).WithTimeout(scaffold.DefaultTimeout).ProbeEvery(scaffold.DefaultInterval).Should(Succeed()) + s.RetryAssertion(func() (string, error) { + return s.GetResourceYaml("Gateway", s.Namespace()) + }).ShouldNot(ContainSubstring("reason: InvalidCertificateRef"), "checking listener condition") + }) + It("Check downstream mTLS via frontendValidation", func() { By("create GatewayProxy") gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index 594da5f6..08e1a991 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -1600,4 +1600,116 @@ spec: }) }) + + // An Ingress has no status conditions to carry a rejection, so it is reported as an + // event instead. + Context("Bad resource isolation", func() { + var gatewayProxy = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config + namespace: %s +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + var ingressClass = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: %s +spec: + controller: "%s" + parameters: + apiGroup: "apisix.apache.org" + kind: "GatewayProxy" + name: "apisix-proxy-config" + namespace: "%s" + scope: "Namespace" +` + // retries has no lower bound in the CRD and the data plane requires it to be at + // least 0, so this reaches the data plane and is rejected on schema grounds. + var rejectedIngress = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-service-e2e-test + namespace: %s +spec: + ingressClassName: %s + retries: -1 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rejected +spec: + ingressClassName: %s + rules: + - host: rejected-ingress.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + + var fixedUpstream = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixUpstream +metadata: + name: httpbin-service-e2e-test + namespace: %s +spec: + ingressClassName: %s + retries: 1 +` + + It("a rejected Ingress is reported as an event", func() { + By("create GatewayProxy") + err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayProxy, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()), s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + + By("create IngressClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClass, s.Namespace(), s.GetControllerName(), s.Namespace()), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass") + + By("create an Ingress whose upstream configuration the data plane rejects") + err = s.CreateResourceFromString(fmt.Sprintf(rejectedIngress, s.Namespace(), s.Namespace(), s.Namespace())) + Expect(err).NotTo(HaveOccurred(), "creating ApisixUpstream and Ingress") + + By("the rejected Ingress reports an event") + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString("events", "--field-selector", "involvedObject.name=rejected", "-n", s.Namespace()) + return output + }).Should(And( + ContainSubstring("Warning"), + ContainSubstring("SyncFailed"), + )) + + By("fix the upstream configuration") + err = s.CreateResourceFromString(fmt.Sprintf(fixedUpstream, s.Namespace(), s.Namespace())) + Expect(err).NotTo(HaveOccurred(), "updating ApisixUpstream") + + By("the Ingress is served") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "rejected-ingress.example.com", + Check: scaffold.WithExpectedStatus(200), + }) + }) + }) })
