This is an automated email from the ASF dual-hosted git repository. rmaucher pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/tomcat.git
commit 15356deebd39e6b60eb3fc15272e21c0d88ab7d0 Author: opencode <[email protected]> AuthorDate: Sun Oct 11 12:50:34 2026 +0200 Fail the start when the Kubernetes service account token cannot be read Reading the service account token at start was wrapped in a log of the failure, after which the service kept running with a provider without a stream provider. No path ever assigns the stream provider again and the fetch short circuits before any token reload could run, so such a node stayed out of the membership for its whole lifetime, while the other nodes saw its pod running through the API and kept replicating session data to it. Replication then went one way only, with the loss of that node sessions on failover as the price, signalled by nothing but one error log at startup. The start now fails closed when the token cannot be read, naming the token path, as the client certificate setup already does for its own errors, so an unusable node is seen instead of trusted. Adapt the storeconfig test, which started a cluster with this provider outside of any pod, to start only the channel receiver and sender. The restrictions are stored with the cluster, so the restart from the written configuration keeps the membership service unstarted too, which the test now asserts. Add a test that an unreadable token fails the start with the io failure as its cause. --- .../cloud/KubernetesMembershipProvider.java | 7 ++- .../membership/cloud/LocalStrings.properties | 1 + .../catalina/storeconfig/TestStoreConfig.java | 9 +++ .../cloud/TestKubernetesMembershipProvider.java | 64 ++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/java/org/apache/catalina/tribes/membership/cloud/KubernetesMembershipProvider.java b/java/org/apache/catalina/tribes/membership/cloud/KubernetesMembershipProvider.java index bdac124c59..b0c76371f5 100644 --- a/java/org/apache/catalina/tribes/membership/cloud/KubernetesMembershipProvider.java +++ b/java/org/apache/catalina/tribes/membership/cloud/KubernetesMembershipProvider.java @@ -101,7 +101,12 @@ public class KubernetesMembershipProvider extends CloudMembershipProvider { streamProvider = new TokenStreamProvider(new String(bytes, StandardCharsets.US_ASCII), caCertFile); saTokenLastModifiedTime = Files.getLastModifiedTime(saTokenPath); } catch (IOException ioe) { - log.error(sm.getString("kubernetesMembershipProvider.streamError"), ioe); + // Without the token, the Kubernetes API cannot be accessed and a provider without a + // stream provider never recovers, keeping this node out of the membership for its + // whole lifetime while the other nodes keep replicating to it. Fail the start + // instead, as the client certificate branch does for its own errors. + throw new IllegalStateException(sm.getString( + "kubernetesMembershipProvider.serviceAccountTokenReadFailed", saTokenFile), ioe); } } else { String clientKeyFile = getEnv("KUBERNETES_CLIENT_KEY_FILE"); diff --git a/java/org/apache/catalina/tribes/membership/cloud/LocalStrings.properties b/java/org/apache/catalina/tribes/membership/cloud/LocalStrings.properties index 30bba20599..274e59630d 100644 --- a/java/org/apache/catalina/tribes/membership/cloud/LocalStrings.properties +++ b/java/org/apache/catalina/tribes/membership/cloud/LocalStrings.properties @@ -38,6 +38,7 @@ kubernetesMembershipProvider.memberError=Error creating member kubernetesMembershipProvider.noKey=Client certificate key was not specified in the environment kubernetesMembershipProvider.noNamespace=Namespace not set kubernetesMembershipProvider.serviceAccountTokenMissing=Service account token not found at [{0}] +kubernetesMembershipProvider.serviceAccountTokenReadFailed=Unable to read the service account token at [{0}]. The cluster cannot start without access to the Kubernetes API. kubernetesMembershipProvider.streamError=Failed to open stream tokenStream.failedConnection=Failed connection to [{0}] diff --git a/test/org/apache/catalina/storeconfig/TestStoreConfig.java b/test/org/apache/catalina/storeconfig/TestStoreConfig.java index ad59d93984..3fd8e2ef3b 100644 --- a/test/org/apache/catalina/storeconfig/TestStoreConfig.java +++ b/test/org/apache/catalina/storeconfig/TestStoreConfig.java @@ -115,6 +115,12 @@ public class TestStoreConfig extends TomcatBaseTest { tc_CloudMembershipService_53.setMembershipProviderClassName("org.apache.catalina.tribes.membership.cloud.KubernetesMembershipProvider"); tc_GroupChannel_52.setMembershipService(tc_CloudMembershipService_53); tc_SimpleTcpCluster_51.setChannel(tc_GroupChannel_52); + // Cloud membership needs the Kubernetes API when it starts. Start the channel's receiver + // and sender only, so the storeconfig round-trip does not depend on the API. The start + // options are not default, so they are stored in server.xml and apply to the restart from + // the written configuration as well. + tc_SimpleTcpCluster_51.setChannelStartOptions( + org.apache.catalina.tribes.Channel.SND_RX_SEQ | org.apache.catalina.tribes.Channel.SND_TX_SEQ); tomcat.getEngine().setCluster(tc_SimpleTcpCluster_51); org.apache.catalina.realm.LockOutRealm tc_LockOutRealm_55 = new org.apache.catalina.realm.LockOutRealm(); @@ -161,6 +167,9 @@ public class TestStoreConfig extends TomcatBaseTest { } Assert.assertTrue(serverXmlDump.contains("StoreConfigLifecycleListener")); Assert.assertTrue(serverXmlDump.contains("UserDatabaseRealm")); + // The cluster must round-trip the channel start options the test had to restrict, as the + // restart from the written configuration must keep the membership service unstarted + Assert.assertTrue(serverXmlDump.contains("channelStartOptions")); Assert.assertTrue(serverXmlDump.contains("SecretKeyCredentialHandler")); Assert.assertTrue(serverXmlDump.contains("certificateKeystorePassword=")); // The attributes of a stored element wrap at 60 columns, so check diff --git a/test/org/apache/catalina/tribes/membership/cloud/TestKubernetesMembershipProvider.java b/test/org/apache/catalina/tribes/membership/cloud/TestKubernetesMembershipProvider.java new file mode 100644 index 0000000000..b54cae7de6 --- /dev/null +++ b/test/org/apache/catalina/tribes/membership/cloud/TestKubernetesMembershipProvider.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.catalina.tribes.membership.cloud; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import org.apache.catalina.tribes.group.GroupChannel; + +public class TestKubernetesMembershipProvider { + + /* + * When the service account token cannot be read, the provider has no way to access the + * Kubernetes API and never recovers, as the stream provider is only created at start. Such a + * node stays out of the membership for its whole lifetime while the other nodes see its pod + * and keep replicating to it, turning replication one way. The start must therefore fail + * closed, like it does when the client certificate setup fails. + */ + @Test + public void testStartFailsWhenServiceAccountTokenUnreadable() throws Exception { + // The provider reads the default token file when the environment does not redirect it, so + // the test needs an environment without the file and without a redirect to another one + Assume.assumeTrue(System.getenv(CloudMembershipProvider.CUSTOM_ENV_PREFIX + "SA_TOKEN_FILE") == null); + Assume.assumeTrue(System.getenv("SA_TOKEN_FILE") == null); + Assume.assumeTrue( + System.getenv(CloudMembershipProvider.CUSTOM_ENV_PREFIX + "CLIENT_CERT_FILE") == null); + Assume.assumeTrue(System.getenv("KUBERNETES_CLIENT_CERTIFICATE_FILE") == null); + Assume.assumeFalse(Files.exists( + FileSystems.getDefault().getPath("/var/run/secrets/kubernetes.io/serviceaccount/token"))); + + GroupChannel channel = new GroupChannel(); + channel.setName("TestKubernetesMembershipProvider-" + System.nanoTime()); + + CloudMembershipService service = new CloudMembershipService(); + service.setChannel(channel); + service.setLocalMemberProperties("127.0.0.1", 4006, 0, 0); + try { + service.start(CloudMembershipService.MBR_RX); + Assert.fail("start() must fail when the service account token cannot be read"); + } catch (IllegalStateException expected) { + // The start fails closed rather than running without access to the API + Assert.assertTrue(expected.getCause() instanceof IOException); + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
