This is an automated email from the ASF dual-hosted git repository. ycai pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-sidecar.git
The following commit(s) were added to refs/heads/trunk by this push: new cf796be CASSANDRASC-63: Support credential rotation in JmxClient cf796be is described below commit cf796be039b9b08364de11fd7fec8a00cf699616 Author: Francisco Guerrero <francisco.guerr...@apple.com> AuthorDate: Tue Jun 27 21:20:13 2023 -0700 CASSANDRASC-63: Support credential rotation in JmxClient JMX credentials in a Cassandra instance can be rotated on a cadence, on every bounce, or by some other means. In those cases, the `JmxClient` will no longer be able to connect to the instance completely losing the ability to talk to that instance. In this commit, we allow the `JmxClient` to support credential changes to be continue to talk to the Cassandra instance uninterrupted without any potential downtime to the Sidecar service. patch by Francisco Guerrero; reviewed by Dinesh Joshi, Yifan Cai for CASSANDRASC-63 # Conflicts: # CHANGES.txt --- CHANGES.txt | 1 + .../apache/cassandra/sidecar/common/JmxClient.java | 63 ++++++++--- .../exceptions/JmxAuthenticationException.java | 70 ++++++++++++ .../cassandra/sidecar/common/JmxClientTest.java | 124 ++++++++++++++++++--- .../cassandra/sidecar/routes/AbstractHandler.java | 20 +++- .../org/apache/cassandra/sidecar/TestModule.java | 2 +- .../apache/cassandra/sidecar/TestSslModule.java | 62 ++++++----- .../cassandra/sidecar/routes/RingHandlerTest.java | 33 ++++-- 8 files changed, 304 insertions(+), 71 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5dc930b..8e5826f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ 1.0.0 ----- + * Support credential rotation in JmxClient (CASSANDRASC-63) * File descriptor is not being closed on MD5 checksum (CASSANDRASC-64) * Expose JMX host and port from JMXClient (CASSANDRASC-59) * Support retries in Sidecar Client on Invalid Checksum (CASSANDRASC-58) diff --git a/common/src/main/java/org/apache/cassandra/sidecar/common/JmxClient.java b/common/src/main/java/org/apache/cassandra/sidecar/common/JmxClient.java index b5db0a5..d5cdcf3 100644 --- a/common/src/main/java/org/apache/cassandra/sidecar/common/JmxClient.java +++ b/common/src/main/java/org/apache/cassandra/sidecar/common/JmxClient.java @@ -25,6 +25,9 @@ import java.rmi.server.RMIClientSocketFactory; import java.rmi.server.RMISocketFactory; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; import javax.management.JMX; import javax.management.MBeanServerConnection; import javax.management.MalformedObjectNameException; @@ -37,6 +40,7 @@ import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; import javax.rmi.ssl.SslRMIClientSocketFactory; +import org.apache.cassandra.sidecar.common.exceptions.JmxAuthenticationException; import org.jetbrains.annotations.VisibleForTesting; /** @@ -49,9 +53,11 @@ public class JmxClient implements NotificationListener, Closeable public static final String REGISTRY_CONTEXT_SOCKET_FACTORY = "com.sun.jndi.rmi.factory.socket"; private final JMXServiceURL jmxServiceURL; private MBeanServerConnection mBeanServerConnection; - private final Map<String, Object> jmxEnv; private boolean connected = false; private JMXConnector jmxConnector; + private final Supplier<String> roleSupplier; + private final Supplier<String> passwordSupplier; + private final BooleanSupplier enableSslSupplier; /** * Creates a new client with the provided {@code host} and {@code port}. @@ -75,32 +81,30 @@ public class JmxClient implements NotificationListener, Closeable */ public JmxClient(String host, int port, String role, String password, boolean enableSSl) { - this(buildJmxServiceURL(host, port), role, password, enableSSl); + this(buildJmxServiceURL(host, port), () -> role, () -> password, () -> enableSSl); } @VisibleForTesting JmxClient(JMXServiceURL jmxServiceURL) { - this(jmxServiceURL, null, null, false); + this(jmxServiceURL, () -> null, () -> null, () -> false); } @VisibleForTesting JmxClient(JMXServiceURL jmxServiceURL, String role, String password) { - this(jmxServiceURL, role, password, false); + this(jmxServiceURL, () -> role, () -> password, () -> false); } - private JmxClient(JMXServiceURL jmxServiceURL, String role, String password, boolean enableSsl) + public JmxClient(JMXServiceURL jmxServiceURL, + Supplier<String> roleSupplier, + Supplier<String> passwordSupplier, + BooleanSupplier enableSslSupplier) { - this.jmxServiceURL = jmxServiceURL; - - jmxEnv = new HashMap<>(); - if (role != null && password != null) - { - String[] credentials = new String[]{ role, password }; - jmxEnv.put(JMXConnector.CREDENTIALS, credentials); - } - jmxEnv.put(REGISTRY_CONTEXT_SOCKET_FACTORY, rmiClientSocketFactory(enableSsl)); + this.jmxServiceURL = Objects.requireNonNull(jmxServiceURL, "jmxServiceURL is required"); + this.roleSupplier = Objects.requireNonNull(roleSupplier, "roleSupplier is required"); + this.passwordSupplier = Objects.requireNonNull(passwordSupplier, "passwordSupplier is required"); + this.enableSslSupplier = Objects.requireNonNull(enableSslSupplier, "enableSslSupplier is required"); } /** @@ -147,7 +151,7 @@ public class JmxClient implements NotificationListener, Closeable { try { - jmxConnector = JMXConnectorFactory.connect(jmxServiceURL, jmxEnv); + jmxConnector = JMXConnectorFactory.connect(jmxServiceURL, buildJmxEnv()); jmxConnector.addConnectionNotificationListener(this, null, null); mBeanServerConnection = jmxConnector.getMBeanServerConnection(); connected = true; @@ -158,6 +162,19 @@ public class JmxClient implements NotificationListener, Closeable throw new RuntimeException(String.format("Failed to connect to JMX endpoint %s", jmxServiceURL), iox); } + catch (SecurityException securityException) + { + connected = false; + String errorMessage = securityException.getMessage() != null + ? securityException.getMessage() + : "JMX Authentication failed"; + throw new JmxAuthenticationException(errorMessage, securityException); + } + catch (RuntimeException runtimeException) + { + // catch exceptions coming from the lambdas and wrap them in a JmxAuthenticationException + throw new JmxAuthenticationException(runtimeException); + } } @Override @@ -216,6 +233,22 @@ public class JmxClient implements NotificationListener, Closeable } } + private Map<String, Object> buildJmxEnv() + { + String role = roleSupplier.get(); + String password = passwordSupplier.get(); + boolean enableSsl = enableSslSupplier.getAsBoolean(); + + Map<String, Object> jmxEnv = new HashMap<>(); + if (role != null && password != null) + { + String[] credentials = new String[]{ role, password }; + jmxEnv.put(JMXConnector.CREDENTIALS, credentials); + } + jmxEnv.put(REGISTRY_CONTEXT_SOCKET_FACTORY, rmiClientSocketFactory(enableSsl)); + return jmxEnv; + } + @Override public synchronized void close() throws IOException { diff --git a/common/src/main/java/org/apache/cassandra/sidecar/common/exceptions/JmxAuthenticationException.java b/common/src/main/java/org/apache/cassandra/sidecar/common/exceptions/JmxAuthenticationException.java new file mode 100644 index 0000000..f09f778 --- /dev/null +++ b/common/src/main/java/org/apache/cassandra/sidecar/common/exceptions/JmxAuthenticationException.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.exceptions; + +/** + * Exceptions encountered during JMX client authentication to the server + */ +public class JmxAuthenticationException extends RuntimeException +{ + /** + * Constructs a JMX authentication exception with the specified detail + * message. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + */ + public JmxAuthenticationException(String message) + { + super(message); + } + + /** + * Constructs a JMX authentication exception with the specified cause + * and a detail message of {@code (cause==null ? null : cause.toString())} + * (which typically contains the class and detail message of + * {@code cause}). This constructor is useful for runtime exceptions + * that are little more than wrappers for other throwables. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public JmxAuthenticationException(Throwable cause) + { + super(cause); + } + + /** + * Constructs a JMX authentication exception with the specified detail + * message and cause. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public JmxAuthenticationException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/common/src/test/java/org/apache/cassandra/sidecar/common/JmxClientTest.java b/common/src/test/java/org/apache/cassandra/sidecar/common/JmxClientTest.java index 4b716f6..e3269f6 100644 --- a/common/src/test/java/org/apache/cassandra/sidecar/common/JmxClientTest.java +++ b/common/src/test/java/org/apache/cassandra/sidecar/common/JmxClientTest.java @@ -32,6 +32,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; import java.util.logging.Logger; import javax.management.MBeanServer; import javax.management.ObjectName; @@ -46,6 +49,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.platform.commons.util.Preconditions; +import org.apache.cassandra.sidecar.common.exceptions.JmxAuthenticationException; + import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -138,23 +143,110 @@ public class JmxClientTest } @Test - public void testCallWithoutCredentialsFails() + public void testCallWithoutCredentialsFails() throws IOException + { + try (JmxClient client = new JmxClient(serviceURL)) + { + assertThatExceptionOfType(JmxAuthenticationException.class) + .isThrownBy(() -> + client.proxy(Import.class, objectName) + .importNewSSTables(Sets.newHashSet("foo", "bar"), + true, + true, + true, + true, + true, + true)) + .withMessageContaining("Authentication failed! Credentials required"); + } + } + + @Test + public void testRoleSupplierThrows() throws IOException + { + String errorMessage = "bad role state!"; + Supplier<String> roleSupplier = () -> { + throw new IllegalStateException(errorMessage); + }; + testSupplierThrows(errorMessage, + new JmxClient(serviceURL, roleSupplier, () -> "password", () -> false)); + } + + @Test + public void testPasswordSupplierThrows() throws IOException + { + String errorMessage = "bad password state!"; + Supplier<String> passwordSupplier = () -> { + throw new IllegalStateException(errorMessage); + }; + testSupplierThrows(errorMessage, + new JmxClient(serviceURL, () -> "controlRole", passwordSupplier, () -> false)); + } + + @Test + public void testEnableSslSupplierThrows() throws IOException + { + String errorMessage = "bad ssl supplier state!"; + BooleanSupplier enableSslSupplier = () -> { + throw new IllegalStateException(errorMessage); + }; + testSupplierThrows(errorMessage, + new JmxClient(serviceURL, () -> "controlRole", () -> "password", enableSslSupplier)); + } + + private static void testSupplierThrows(String errorMessage, JmxClient jmxClient) throws IOException { - assertThatExceptionOfType(SecurityException.class) - .isThrownBy(() -> - { - try (JmxClient client = new JmxClient(serviceURL)) - { - client.proxy(Import.class, objectName) - .importNewSSTables(Sets.newHashSet("foo", "bar"), - true, - true, - true, - true, - true, - true); - } - }); + try (JmxClient client = jmxClient) + { + assertThatExceptionOfType(JmxAuthenticationException.class) + .isThrownBy(() -> + client.proxy(Import.class, objectName) + .importNewSSTables(Sets.newHashSet("foo", "bar"), + true, + true, + true, + true, + true, + true)) + .withMessageContaining(errorMessage); + } + } + + @Test + public void testRetryAfterAuthenticationFailureWithCorrectCredentials() throws IOException + { + AtomicInteger tryCount = new AtomicInteger(0); + List<String> result; + Supplier<String> passwordSupplier = () -> { + if (tryCount.getAndIncrement() == 0) + { + // authentication fails on the first attempt + return "bad password"; + } + return "password"; + }; + try (JmxClient client = new JmxClient(serviceURL, () -> "controlRole", passwordSupplier, () -> false)) + { + // First attempt fails + assertThatExceptionOfType(JmxAuthenticationException.class) + .isThrownBy(() -> + client.proxy(Import.class, objectName) + .importNewSSTables(Sets.newHashSet("foo", "bar"), + true, + true, + true, + true, + true, + true)) + .withMessageContaining("Authentication failed! Invalid username or password"); + + // second attempt succeeds after getting the correct password + result = client.proxy(Import.class, objectName) + .importNewSSTables(Sets.newHashSet("foo", "bar"), true, + true, true, true, true, + true); + } + assertThat(result.size()).isEqualTo(0); } @Test diff --git a/src/main/java/org/apache/cassandra/sidecar/routes/AbstractHandler.java b/src/main/java/org/apache/cassandra/sidecar/routes/AbstractHandler.java index 78c8b24..12cd48a 100644 --- a/src/main/java/org/apache/cassandra/sidecar/routes/AbstractHandler.java +++ b/src/main/java/org/apache/cassandra/sidecar/routes/AbstractHandler.java @@ -32,6 +32,7 @@ import io.vertx.ext.web.RoutingContext; import io.vertx.ext.web.handler.HttpException; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.data.QualifiedTableName; +import org.apache.cassandra.sidecar.common.exceptions.JmxAuthenticationException; import org.apache.cassandra.sidecar.common.utils.CassandraInputValidator; import org.apache.cassandra.sidecar.concurrent.ExecutorPools; import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher; @@ -165,9 +166,7 @@ public abstract class AbstractHandler<T> implements Handler<RoutingContext> protected void processFailure(Throwable cause, RoutingContext context, String host, SocketAddress remoteAddress, T request) { - HttpException httpException = cause instanceof HttpException - ? (HttpException) cause - : wrapHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, cause); + HttpException httpException = determineHttpException(cause); if (HttpStatusClass.CLIENT_ERROR.contains(httpException.getStatusCode())) { @@ -183,6 +182,21 @@ public abstract class AbstractHandler<T> implements Handler<RoutingContext> context.fail(httpException); } + protected HttpException determineHttpException(Throwable cause) + { + if (cause instanceof HttpException) + { + return (HttpException) cause; + } + + if (cause instanceof JmxAuthenticationException) + { + return wrapHttpException(HttpResponseStatus.SERVICE_UNAVAILABLE, cause); + } + + return wrapHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, cause); + } + /** * Returns the validated {@link QualifiedTableName} from the context, where the both keyspace and table name * are required. diff --git a/src/test/java/org/apache/cassandra/sidecar/TestModule.java b/src/test/java/org/apache/cassandra/sidecar/TestModule.java index 1573972..4723944 100644 --- a/src/test/java/org/apache/cassandra/sidecar/TestModule.java +++ b/src/test/java/org/apache/cassandra/sidecar/TestModule.java @@ -64,7 +64,7 @@ public class TestModule extends AbstractModule protected Configuration abstractConfig(InstancesConfig instancesConfig) { WorkerPoolConfiguration workPoolConf = new WorkerPoolConfiguration("test-pool", 10, 30000); - return new Configuration.Builder() + return new Configuration.Builder<>() .setInstancesConfig(instancesConfig) .setHost("127.0.0.1") .setPort(6475) diff --git a/src/test/java/org/apache/cassandra/sidecar/TestSslModule.java b/src/test/java/org/apache/cassandra/sidecar/TestSslModule.java index 3ed4e7e..5e39e83 100644 --- a/src/test/java/org/apache/cassandra/sidecar/TestSslModule.java +++ b/src/test/java/org/apache/cassandra/sidecar/TestSslModule.java @@ -18,7 +18,8 @@ package org.apache.cassandra.sidecar; -import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,46 +35,49 @@ public class TestSslModule extends TestModule { private static final Logger logger = LoggerFactory.getLogger(TestSslModule.class); - @Override public Configuration abstractConfig(InstancesConfig instancesConfig) { - final String keyStorePath = TestSslModule.class.getClassLoader().getResource("certs/test.p12").getPath(); - final String keyStorePassword = "password"; + String keyStorePath = TestSslModule.class.getClassLoader() + .getResource("certs/test.p12") + .getPath(); + String keyStorePassword = "password"; - final String trustStorePath = TestSslModule.class.getClassLoader().getResource("certs/ca.p12").getPath(); - final String trustStorePassword = "password"; + String trustStorePath = TestSslModule.class.getClassLoader() + .getResource("certs/ca.p12") + .getPath(); + String trustStorePassword = "password"; - if (!new File(keyStorePath).exists()) + if (!Files.exists(Paths.get(keyStorePath))) { - logger.error("JMX password file not found"); + logger.error("JMX password file not found in path={}", keyStorePath); } - if (!new File(trustStorePath).exists()) + if (!Files.exists(Paths.get(trustStorePath))) { - logger.error("Trust Store file not found"); + logger.error("Trust Store file not found in path={}", trustStorePath); } WorkerPoolConfiguration workerPoolConf = new WorkerPoolConfiguration("test-pool", 10, 30000); - return new Configuration.Builder() - .setInstancesConfig(instancesConfig) - .setHost("127.0.0.1") - .setPort(6475) - .setHealthCheckFrequency(1000) - .setKeyStorePath(keyStorePath) - .setKeyStorePassword(keyStorePassword) - .setTrustStorePath(trustStorePath) - .setTrustStorePassword(trustStorePassword) - .setSslEnabled(true) - .setRateLimitStreamRequestsPerSecond(1) - .setRequestIdleTimeoutMillis(300_000) - .setRequestTimeoutMillis(300_000L) - .setConcurrentUploadsLimit(80) - .setMinSpacePercentRequiredForUploads(0) - .setSSTableImportCacheConfiguration(new CacheConfiguration(60_000, 100)) - .setServerWorkerPoolConfiguration(workerPoolConf) - .setServerInternalWorkerPoolConfiguration(workerPoolConf) - .build(); + return new Configuration.Builder<>() + .setInstancesConfig(instancesConfig) + .setHost("127.0.0.1") + .setPort(6475) + .setHealthCheckFrequency(1000) + .setKeyStorePath(keyStorePath) + .setKeyStorePassword(keyStorePassword) + .setTrustStorePath(trustStorePath) + .setTrustStorePassword(trustStorePassword) + .setSslEnabled(true) + .setRateLimitStreamRequestsPerSecond(1) + .setRequestIdleTimeoutMillis(300_000) + .setRequestTimeoutMillis(300_000L) + .setConcurrentUploadsLimit(80) + .setMinSpacePercentRequiredForUploads(0) + .setSSTableImportCacheConfiguration(new CacheConfiguration(60_000, 100)) + .setServerWorkerPoolConfiguration(workerPoolConf) + .setServerInternalWorkerPoolConfiguration(workerPoolConf) + .build(); } } diff --git a/src/test/java/org/apache/cassandra/sidecar/routes/RingHandlerTest.java b/src/test/java/org/apache/cassandra/sidecar/routes/RingHandlerTest.java index d0467dd..73631e8 100644 --- a/src/test/java/org/apache/cassandra/sidecar/routes/RingHandlerTest.java +++ b/src/test/java/org/apache/cassandra/sidecar/routes/RingHandlerTest.java @@ -47,7 +47,6 @@ import io.vertx.ext.web.client.WebClient; import io.vertx.ext.web.client.predicate.ResponsePredicate; import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; -import org.apache.cassandra.sidecar.Configuration; import org.apache.cassandra.sidecar.MainModule; import org.apache.cassandra.sidecar.TestModule; import org.apache.cassandra.sidecar.cluster.InstancesConfig; @@ -56,6 +55,7 @@ import org.apache.cassandra.sidecar.common.CassandraAdapterDelegate; import org.apache.cassandra.sidecar.common.StorageOperations; import org.apache.cassandra.sidecar.common.data.RingEntry; import org.apache.cassandra.sidecar.common.data.RingResponse; +import org.apache.cassandra.sidecar.common.exceptions.JmxAuthenticationException; import org.mockito.stubbing.Answer; import static io.netty.handler.codec.http.HttpResponseStatus.OK; @@ -72,7 +72,6 @@ class RingHandlerTest { static final Logger LOGGER = LoggerFactory.getLogger(RingHandlerTest.class); Vertx vertx; - Configuration config; HttpServer server; @BeforeEach @@ -84,9 +83,8 @@ class RingHandlerTest .with(testOverride)); vertx = injector.getInstance(Vertx.class); server = injector.getInstance(HttpServer.class); - config = injector.getInstance(Configuration.class); VertxTestContext context = new VertxTestContext(); - server.listen(config.getPort(), config.getHost(), context.succeedingThenComplete()); + server.listen(0, "127.0.0.1", context.succeedingThenComplete()); context.awaitCompletion(5, TimeUnit.SECONDS); } @@ -109,7 +107,7 @@ class RingHandlerTest VertxTestContext context = new VertxTestContext(); WebClient client = WebClient.create(vertx); String testRoute = "/api/v1/cassandra/ring/keyspaces/" + keyspace; - client.get(config.getPort(), config.getHost(), testRoute) + client.get(server.actualPort(), "127.0.0.1", testRoute) .expect(ResponsePredicate.SC_BAD_REQUEST) .send(context.succeedingThenComplete()); } @@ -135,7 +133,7 @@ class RingHandlerTest }; WebClient client = WebClient.create(vertx); String testRoute = "/api/v1/cassandra/ring"; - client.get(config.getPort(), config.getHost(), testRoute) + client.get(server.actualPort(), "127.0.0.1", testRoute) .expect(ResponsePredicate.SC_OK) .send(context.succeeding(response -> { assertThat(response.statusCode()).isEqualTo(OK.code()); @@ -168,7 +166,7 @@ class RingHandlerTest }; WebClient client = WebClient.create(vertx); String testRoute = "/api/v1/cassandra/ring"; - client.get(config.getPort(), config.getHost(), testRoute) + client.get(server.actualPort(), "127.0.0.1", testRoute) .expect(ResponsePredicate.SC_INTERNAL_SERVER_ERROR) .send(context.succeeding(response -> { JsonObject error = response.bodyAsJsonObject(); @@ -179,6 +177,27 @@ class RingHandlerTest })); } + @Test + void testGetRingFailsOnUnavailableJmxConnectivity(VertxTestContext context) + { + String errorMessage = "Authentication failed! Invalid username or password"; + RingHandlerTestModule.ringSupplier = () -> { + throw new JmxAuthenticationException(errorMessage); + }; + WebClient client = WebClient.create(vertx); + String testRoute = "/api/v1/cassandra/ring"; + client.get(server.actualPort(), "127.0.0.1", testRoute) + .expect(ResponsePredicate.SC_SERVICE_UNAVAILABLE) + .send(context.succeeding(response -> { + JsonObject error = response.bodyAsJsonObject(); + assertThat(error.getInteger("code")).isEqualTo(503); + assertThat(error.getString("status")).isEqualTo("Service Unavailable"); + assertThat(error.getString("message").contains(errorMessage)); + + context.completeNow(); + })); + } + static class RingHandlerTestModule extends AbstractModule { static Supplier<RingResponse> ringSupplier; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org