IGNITE-9040 StopNodeFailureHandler is not able to stop node correctly on node segmentation
Signed-off-by: Andrey Gura <ag...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/469aaba5 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/469aaba5 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/469aaba5 Branch: refs/heads/ignite-8446 Commit: 469aaba59c0539507972f4725642b2f2f81c08a0 Parents: 49d7a24 Author: Sergey Chugunov <sergey.chugu...@gmail.com> Authored: Wed Jul 25 16:26:12 2018 +0300 Committer: Andrey Gura <ag...@apache.org> Committed: Wed Jul 25 16:26:41 2018 +0300 ---------------------------------------------------------------------- .../ignite/internal/GridKernalContext.java | 7 + .../ignite/internal/GridKernalContextImpl.java | 14 +- .../org/apache/ignite/internal/IgnitionEx.java | 13 +- .../wal/reader/StandaloneGridKernalContext.java | 5 + .../zk/internal/ZookeeperDiscoverySpiTest.java | 139 ++++++++++++++++++- 5 files changed, 164 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java index 505c3d6..051978c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java @@ -445,6 +445,13 @@ public interface GridKernalContext extends Iterable<GridComponent> { public boolean invalid(); /** + * Checks whether this node detected its segmentation from the rest of the grid. + * + * @return {@code True} if this node has segmented, {@code false} otherwise. + */ + public boolean segmented(); + + /** * Gets failure processor. */ public FailureProcessor failure(); http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java index 2f4ecbc..2be64e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java @@ -37,6 +37,7 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager; import org.apache.ignite.internal.managers.collision.GridCollisionManager; import org.apache.ignite.internal.managers.communication.GridIoManager; @@ -1125,7 +1126,18 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable @Override public boolean invalid() { FailureProcessor failureProc = failure(); - return failureProc != null && failureProc.failureContext() != null; + return failureProc != null + && failureProc.failureContext() != null + && failureProc.failureContext().type() != FailureType.SEGMENTATION; + } + + /** {@inheritDoc} */ + @Override public boolean segmented() { + FailureProcessor failureProc = failure(); + + return failureProc != null + && failureProc.failureContext() != null + && failureProc.failureContext().type() == FailureType.SEGMENTATION; } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index f90fe82..aff19c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -2604,15 +2604,12 @@ public class IgnitionEx { throw e; } finally { - if (!grid0.context().invalid()) + if (grid0.context().segmented()) + state = STOPPED_ON_SEGMENTATION; + else if (grid0.context().invalid()) + state = STOPPED_ON_FAILURE; + else state = STOPPED; - else { - FailureContext failure = grid0.context().failure().failureContext(); - - state = failure.type() == FailureType.SEGMENTATION ? - STOPPED_ON_SEGMENTATION : - STOPPED_ON_FAILURE; - } grid = null; http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index 795d460..b9ab76a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -472,6 +472,11 @@ public class StandaloneGridKernalContext implements GridKernalContext { } /** {@inheritDoc} */ + @Override public boolean segmented() { + return false; + } + + /** {@inheritDoc} */ @Override public FailureProcessor failure() { return null; } http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java index 4ccafc3..077c4da 100644 --- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java @@ -59,6 +59,8 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.IgniteState; +import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.ClusterNode; @@ -127,6 +129,7 @@ import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiMBean; import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZKUtil; import org.apache.zookeeper.ZkTestClientCnxnSocketNIO; @@ -147,6 +150,9 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTAN import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2; import static org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD; +import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; import static org.apache.zookeeper.ZooKeeper.ZOOKEEPER_CLIENT_CNXN_SOCKET; /** @@ -182,6 +188,12 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { private boolean testSockNio; /** */ + private CacheAtomicityMode atomicityMode; + + /** */ + private int backups = -1; + + /** */ private boolean testCommSpi; /** */ @@ -278,11 +290,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { cfg.setDiscoverySpi(zkSpi); - CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); - - ccfg.setWriteSynchronizationMode(FULL_SYNC); - - cfg.setCacheConfiguration(ccfg); + cfg.setCacheConfiguration(getCacheConfiguration()); Boolean clientMode = clientThreadLoc.get(); @@ -366,6 +374,21 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { return cfg; } + /** */ + private CacheConfiguration getCacheConfiguration() { + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + ccfg.setWriteSynchronizationMode(FULL_SYNC); + + if (atomicityMode != null) + ccfg.setAtomicityMode(atomicityMode); + + if (backups > 0) + ccfg.setBackups(backups); + + return ccfg; + } + /** * @param clientMode Client mode flag for started nodes. */ @@ -1075,6 +1098,112 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { } /** + * Verifies correct handling of SEGMENTATION event with STOP segmentation policy: node is stopped successfully, + * all its threads are shut down. + * + * @throws Exception If failed. + * + * @see <a href="https://issues.apache.org/jira/browse/IGNITE-9040";>IGNITE-9040</a> ticket for more context of the test. + */ + public void testStopNodeOnSegmentaion() throws Exception { + try { + System.setProperty("IGNITE_WAL_LOG_TX_RECORDS", "true"); + + sesTimeout = 2000; + testSockNio = true; + persistence = true; + atomicityMode = CacheAtomicityMode.TRANSACTIONAL; + backups = 2; + + final Ignite node0 = startGrid(0); + + sesTimeout = 10_000; + testSockNio = false; + + startGrid(1); + + node0.cluster().active(true); + + clientMode(true); + + final IgniteEx client = startGrid(2); + + //first transaction + client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 0, 0); + client.cache(DEFAULT_CACHE_NAME).put(0, 0); + + //second transaction to create a deadlock with the first one + // and guarantee transaction futures will be presented on segmented node + // (erroneous write to WAL on segmented node stop happens + // on completing transaction with NodeStoppingException) + GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + Transaction tx2 = client.transactions().txStart(OPTIMISTIC, READ_COMMITTED, 0, 0); + client.cache(DEFAULT_CACHE_NAME).put(0, 0); + tx2.commit(); + } + }); + + //next block simulates Ignite node segmentation by closing socket of ZooKeeper client + { + final CountDownLatch l = new CountDownLatch(1); + + node0.events().localListen(new IgnitePredicate<Event>() { + @Override public boolean apply(Event evt) { + l.countDown(); + + return false; + } + }, EventType.EVT_NODE_SEGMENTED); + + ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0); + + c0.closeSocket(true); + + for (int i = 0; i < 10; i++) { + Thread.sleep(1_000); + + if (l.getCount() == 0) + break; + } + + info("Allow connect"); + + c0.allowConnect(); + + assertTrue(l.await(10, TimeUnit.SECONDS)); + } + + waitForNodeStop(node0.name()); + + checkStoppedNodeThreads(node0.name()); + } + finally { + System.clearProperty("IGNITE_WAL_LOG_TX_RECORDS"); + } + } + + /** */ + private void checkStoppedNodeThreads(String nodeName) { + Set<Thread> threads = Thread.getAllStackTraces().keySet(); + + for (Thread t : threads) { + if (t.getName().contains(nodeName)) + throw new AssertionError("Thread from stopped node has been found: " + t.getName()); + } + } + + /** */ + private void waitForNodeStop(String name) throws Exception { + while (true) { + if (IgnitionEx.state(name).equals(IgniteState.STARTED)) + Thread.sleep(2000); + else + break; + } + } + + /** * @throws Exception If failed. */ public void testSegmentation1() throws Exception {
