This is an automated email from the ASF dual-hosted git repository.
ilyak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new d75bfd0 IGNITE-13655 Implement readiness probe REST endpoint - Fixes
#8417.
d75bfd0 is described below
commit d75bfd0f35d43ab29f9b280d31e9a0c3132da59d
Author: akorensh <akore...@gmail.com>
AuthorDate: Fri Nov 13 13:51:28 2020 +0300
IGNITE-13655 Implement readiness probe REST endpoint - Fixes #8417.
Signed-off-by: Ilya Kasnacheev <ilya.kasnach...@gmail.com>
---
.../internal/client/rest/GridProbeCommandTest.java | 224 +++++++++++++++++++++
.../client/suite/IgniteClientTestSuite.java | 4 +
.../org/apache/ignite/internal/IgnitionEx.java | 17 ++
.../internal/processors/rest/GridRestCommand.java | 5 +-
.../processors/rest/GridRestProcessor.java | 2 +
.../internal/processors/rest/GridRestResponse.java | 3 +
.../handlers/probe/GridProbeCommandHandler.java | 70 +++++++
.../protocols/http/jetty/GridJettyRestHandler.java | 5 +-
8 files changed, 327 insertions(+), 3 deletions(-)
diff --git
a/modules/clients/src/test/java/org/apache/ignite/internal/client/rest/GridProbeCommandTest.java
b/modules/clients/src/test/java/org/apache/ignite/internal/client/rest/GridProbeCommandTest.java
new file mode 100644
index 0000000..af09544
--- /dev/null
+++
b/modules/clients/src/test/java/org/apache/ignite/internal/client/rest/GridProbeCommandTest.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.client.rest;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.rest.GridRestCommand;
+import org.apache.ignite.internal.processors.rest.GridRestResponse;
+import
org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandler;
+import
org.apache.ignite.internal.processors.rest.handlers.probe.GridProbeCommandHandler;
+import org.apache.ignite.internal.processors.rest.request.GridRestCacheRequest;
+import org.apache.ignite.plugin.AbstractTestPluginProvider;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Test whether REST probe command works correctly when kernal has started and
vice versa.
+ */
+public class GridProbeCommandTest extends GridCommonAbstractTest {
+ /** */
+ private static final int JETTY_PORT = 8080;
+
+ /** */
+ private CountDownLatch triggerRestCmdLatch = new CountDownLatch(1);
+
+ /** */
+ private CountDownLatch triggerPluginStartLatch = new CountDownLatch(1);
+
+ /** */
+ public static Map<String, Object> executeProbeRestRequest() throws
IOException {
+ HttpURLConnection conn = (HttpURLConnection)(new
URL("http://localhost:"; + JETTY_PORT + "/ignite?cmd=probe").openConnection());
+ conn.connect();
+
+ boolean isHTTP_OK = conn.getResponseCode() ==
HttpURLConnection.HTTP_OK;
+
+ Map<String, Object> restResponse = null;
+
+ try (InputStreamReader streamReader = new InputStreamReader(isHTTP_OK
? conn.getInputStream() : conn.getErrorStream())) {
+
+ ObjectMapper objMapper = new ObjectMapper();
+ restResponse = objMapper.readValue(streamReader,
+ new TypeReference<Map<String, Object>>() {
+ });
+
+ log.info("probe command response is: " + restResponse);
+
+ }
+ catch (Exception e) {
+ log.error("error executing probe rest command", e);
+ }
+ return restResponse;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+ cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+ if (igniteInstanceName.equals("regular"))
+ return cfg;
+ else if (igniteInstanceName.equals("delayedStart")) {
+ PluginProvider delayedStartPluginProvider = new
DelayedStartPluginProvider(triggerPluginStartLatch, triggerRestCmdLatch);
+
+ cfg.setPluginProviders(new PluginProvider[]
{delayedStartPluginProvider});
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids(false);
+ }
+
+ /**
+ * Test for the REST probe command
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testRestProbeCommand() throws Exception {
+ startGrid("regular");
+
+ GridRestCommandHandler hnd = new
GridProbeCommandHandler((grid("regular")).context());
+
+ GridRestCacheRequest req = new GridRestCacheRequest();
+ req.command(GridRestCommand.PROBE);
+
+ IgniteInternalFuture<GridRestResponse> resp = hnd.handleAsync(req);
+ resp.get();
+
+ assertEquals(GridRestResponse.STATUS_SUCCESS,
resp.result().getSuccessStatus());
+ assertEquals("grid has started", resp.result().getResponse());
+
+ }
+
+ /**
+ * <p>Test rest cmd=probe command given a non fully started kernal. </p>
+ * <p>1. start the grid on a seperate thread w/a plugin that will keep it
waiting, at a point after rest http
+ * processor is ready, until signaled to proceed. </p>
+ * <p>2. when the grid.start() has reached the plugin init method(rest
http processor has started now), issue a
+ * rest command against the non-fully started kernal. </p>
+ * <p>3. validate that the probe cmd has returned the appropriate
erroneous code and message. </p>
+ * <p>4. stop the grid. </p>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testRestProbeCommandGridNotStarted() throws Exception {
+ new Thread(new Runnable() {
+ @Override public void run() {
+ try {
+ startGrid("delayedStart");
+ }
+ catch (Exception e) {
+ log.error("error when starting delatedStart grid", e);
+ }
+ }
+ }).start();
+
+ Map<String, Object> probeRestCommandResponse;
+
+ log.info("awaiting plugin handler latch");
+ triggerPluginStartLatch.await();
+ log.info("starting rest command url call");
+ try {
+ probeRestCommandResponse = executeProbeRestRequest();
+ log.info("finished rest command url call");
+ }
+ finally {
+ triggerRestCmdLatch.countDown(); //make sure the grid shuts down
+ }
+
+ assertTrue(probeRestCommandResponse.get("error").equals("grid has not
started"));
+ assertEquals(GridRestResponse.SERVICE_UNAVAILABLE,
probeRestCommandResponse.get("successStatus"));
+ }
+
+ /**
+ * <p>Start a regular grid, issue a cmd=probe rest command, and validate
restponse
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testRestProbeCommandGridStarted() throws Exception {
+ startGrid("regular");
+
+ Map<String, Object> probeRestCommandResponse;
+
+ probeRestCommandResponse = executeProbeRestRequest();
+
+ assertTrue(probeRestCommandResponse.get("response").equals("grid has
started"));
+ assertEquals(0, probeRestCommandResponse.get("successStatus"));
+ }
+
+ /**
+ * This plugin awaits until it is given the signal to process -- thereby
allowing an http request against a non
+ * fully started kernal.
+ */
+ public static class DelayedStartPluginProvider extends
AbstractTestPluginProvider {
+ /** */
+ private CountDownLatch triggerRestCmd;
+
+ /** */
+ private CountDownLatch triggerPluginStart;
+
+ /** */
+ public DelayedStartPluginProvider(CountDownLatch
triggerPluginStartLatch,
+ CountDownLatch triggerRestCmdLatch) {
+ this.triggerPluginStart = triggerPluginStartLatch;
+ this.triggerRestCmd = triggerRestCmdLatch;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return "DelayedStartPlugin";
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onIgniteStart() {
+ super.onIgniteStart();
+
+ triggerPluginStart.countDown();
+
+ log.info("awaiting rest command latch ...");
+
+ try {
+ triggerRestCmd.await();
+ }
+ catch (InterruptedException e) {
+ log.error("error in custom plugin", e);
+ }
+
+ log.info("finished awaiting rest command latch.");
+ }
+ }
+}
diff --git
a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
index 8a3936b..7408f4e 100644
---
a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
+++
b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
@@ -42,6 +42,7 @@ import
org.apache.ignite.internal.client.integration.ClientTcpSslDirectSelfTest;
import
org.apache.ignite.internal.client.integration.ClientTcpSslMultiNodeSelfTest;
import org.apache.ignite.internal.client.integration.ClientTcpSslSelfTest;
import
org.apache.ignite.internal.client.integration.ClientTcpUnreachableMultiNodeSelfTest;
+import org.apache.ignite.internal.client.rest.GridProbeCommandTest;
import org.apache.ignite.internal.client.router.ClientFailedInitSelfTest;
import org.apache.ignite.internal.client.router.RouterFactorySelfTest;
import org.apache.ignite.internal.client.router.TcpRouterMultiNodeSelfTest;
@@ -134,6 +135,9 @@ import org.junit.runners.Suite;
ClientTcpUnreachableMultiNodeSelfTest.class,
ClientPreferDirectSelfTest.class,
+ //Test REST probe cmd
+ GridProbeCommandTest.class,
+
// Test client with many nodes and in multithreaded scenarios
ClientTcpMultiThreadedSelfTest.class,
ClientTcpSslMultiThreadedSelfTest.class,
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index dfdae46..3104269 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -1477,6 +1477,16 @@ public class IgnitionEx {
}
/**
+ * @param name Grid name (possibly {@code null} for default grid).
+ * @return true when all managers, processors, and plugins have started
and ignite kernal start method has fully
+ * completed.
+ */
+ public static boolean hasKernalStarted(String name) {
+ IgniteNamedInstance grid = name != null ? grids.get(name) : dfltGrid;
+ return grid != null && grid.hasStartLatchCompleted();
+ }
+
+ /**
* Start context encapsulates all starting parameters.
*/
private static final class GridStartContext {
@@ -3215,6 +3225,13 @@ public class IgnitionEx {
this.cnt = cnt;
}
}
+
+ /**
+ * @return whether the startLatch has been counted down, thereby
indicating that the kernal has full started.
+ */
+ public boolean hasStartLatchCompleted() {
+ return startLatch.getCount() == 0;
+ }
}
/**
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
index c97c26a..16dc5f0 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
@@ -223,7 +223,10 @@ public enum GridRestCommand {
NODE_STATE_BEFORE_START("nodestatebeforestart"),
/** Warm-up. */
- WARM_UP("warmup");
+ WARM_UP("warmup"),
+
+ /** probe. */
+ PROBE("probe");
/** Enum values. */
private static final GridRestCommand[] VALS = values();
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index 358f75b..21c5eb3 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -56,6 +56,7 @@ import
org.apache.ignite.internal.processors.rest.handlers.cluster.GridClusterNa
import
org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructuresCommandHandler;
import
org.apache.ignite.internal.processors.rest.handlers.log.GridLogCommandHandler;
import
org.apache.ignite.internal.processors.rest.handlers.memory.MemoryMetricsCommandHandler;
+import
org.apache.ignite.internal.processors.rest.handlers.probe.GridProbeCommandHandler;
import
org.apache.ignite.internal.processors.rest.handlers.query.QueryCommandHandler;
import
org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler;
import
org.apache.ignite.internal.processors.rest.handlers.top.GridTopologyCommandHandler;
@@ -557,6 +558,7 @@ public class GridRestProcessor extends GridProcessorAdapter
implements IgniteRes
addHandler(new GridBaselineCommandHandler(ctx));
addHandler(new MemoryMetricsCommandHandler(ctx));
addHandler(new NodeStateBeforeStartCommandHandler(ctx));
+ addHandler(new GridProbeCommandHandler(ctx));
// Start protocols.
startTcpProtocol();
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
index 0c3ac04..adefd9e 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestResponse.java
@@ -51,6 +51,9 @@ public class GridRestResponse implements Externalizable {
/** Success status. */
private int successStatus = STATUS_SUCCESS;
+ /** HTTP REQUEST not allowed */
+ public static final int SERVICE_UNAVAILABLE = 503;
+
/** Session token. */
private byte[] sesTokBytes;
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/probe/GridProbeCommandHandler.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/probe/GridProbeCommandHandler.java
new file mode 100644
index 0000000..844dd5b
--- /dev/null
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/probe/GridProbeCommandHandler.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.rest.handlers.probe;
+
+import java.util.Collection;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.processors.rest.GridRestCommand;
+import org.apache.ignite.internal.processors.rest.GridRestResponse;
+import
org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandlerAdapter;
+import org.apache.ignite.internal.processors.rest.request.GridRestRequest;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.PROBE;
+
+/**
+ * Handler for {@link GridRestCommand#PROBE}.
+ */
+public class GridProbeCommandHandler extends GridRestCommandHandlerAdapter {
+ /**
+ * @param ctx Context.
+ */
+ public GridProbeCommandHandler(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** Supported commands. */
+ private static final Collection<GridRestCommand> SUPPORTED_COMMANDS =
U.sealList(PROBE);
+
+ /** {@inheritDoc} */
+ @Override public Collection<GridRestCommand> supportedCommands() {
+ return SUPPORTED_COMMANDS;
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<GridRestResponse>
handleAsync(GridRestRequest req) {
+ assert req != null;
+
+ assert SUPPORTED_COMMANDS.contains(req.command());
+
+ switch (req.command()) {
+ case PROBE: {
+ if (log.isDebugEnabled())
+ log.debug("probe command handler invoked.");
+
+ return new
GridFinishedFuture<>(IgnitionEx.hasKernalStarted(ctx.igniteInstanceName()) ?
new GridRestResponse("grid has started") : new
GridRestResponse(GridRestResponse.SERVICE_UNAVAILABLE, "grid has not started"));
+
+ }
+ }
+
+ return new GridFinishedFuture<>();
+ }
+}
diff --git
a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
index 42b8a30..421bf78 100644
---
a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
+++
b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
@@ -464,7 +464,7 @@ public class GridJettyRestHandler extends AbstractHandler {
if (sesTok != null)
cmdRes.setSessionToken(U.byteArray2HexString(sesTok));
- res.setStatus(HttpServletResponse.SC_OK);
+ res.setStatus(cmdRes.getSuccessStatus() ==
GridRestResponse.SERVICE_UNAVAILABLE ?
HttpServletResponse.SC_SERVICE_UNAVAILABLE : HttpServletResponse.SC_OK);
}
catch (Throwable e) {
res.setStatus(HttpServletResponse.SC_OK);
@@ -720,7 +720,8 @@ public class GridJettyRestHandler extends AbstractHandler {
case DATA_REGION_METRICS:
case DATA_STORAGE_METRICS:
case NAME:
- case VERSION: {
+ case VERSION:
+ case PROBE: {
restReq = new GridRestRequest();
break;
