IGNITE-10262: MVCC TX: Client operation may hangs if all data nodes left the grid. This closes #5693.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7b6e17b2 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7b6e17b2 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7b6e17b2 Branch: refs/heads/ignite-1741 Commit: 7b6e17b216d82e69610da0432b1b37b70a5b8492 Parents: 664fb93 Author: rkondakov <kondako...@mail.ru> Authored: Wed Dec 19 15:50:31 2018 +0300 Committer: Igor Seliverstov <gvvinbl...@gmail.com> Committed: Wed Dec 19 15:50:31 2018 +0300 ---------------------------------------------------------------------- .../near/GridNearTxEnlistFuture.java | 3 +- .../near/GridNearTxQueryEnlistFuture.java | 5 + .../GridNearTxQueryResultsEnlistFuture.java | 3 +- .../IgniteClientCacheStartFailoverTest.java | 4 +- .../processors/query/h2/IgniteH2Indexing.java | 12 +- ...eMvccAbstractSqlCoordinatorFailoverTest.java | 161 +++++++++++++++++++ 6 files changed, 181 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/7b6e17b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java index 789cd3c..8cbe93e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java @@ -31,6 +31,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.cache.CacheEntryPredicate; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMessage; @@ -217,7 +218,7 @@ public class GridNearTxEnlistFuture extends GridNearTxAbstractEnlistFuture<GridC ClusterNode node = cctx.affinity().primaryByKey(key, topVer); if (node == null) - throw new ClusterTopologyCheckedException("Failed to get primary node " + + throw new ClusterTopologyServerNotFoundException("Failed to get primary node " + "[topVer=" + topVer + ", key=" + key + ']'); tx.markQueryEnlisted(null); http://git-wip-us.apache.org/repos/asf/ignite/blob/7b6e17b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java index 030a432..245ef39 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java @@ -23,6 +23,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.affinity.AffinityAssignment; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMessage; @@ -119,6 +120,10 @@ public class GridNearTxQueryEnlistFuture extends GridNearTxQueryAbstractEnlistFu updateMappings(pNode); } + if (primary.isEmpty()) + throw new ClusterTopologyServerNotFoundException("Failed to find data nodes for cache (all partition " + + "nodes left the grid)."); + boolean locallyMapped = primary.contains(cctx.localNode()); if (locallyMapped) http://git-wip-us.apache.org/repos/asf/ignite/blob/7b6e17b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java index 316aae5..e4d74f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java @@ -31,6 +31,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMessage; import org.apache.ignite.internal.processors.cache.KeyCacheObject; @@ -203,7 +204,7 @@ public class GridNearTxQueryResultsEnlistFuture extends GridNearTxQueryAbstractE ClusterNode node = cctx.affinity().primaryByPartition(key.partition(), topVer); if (node == null) - throw new ClusterTopologyCheckedException("Failed to get primary node " + + throw new ClusterTopologyServerNotFoundException("Failed to get primary node " + "[topVer=" + topVer + ", key=" + key + ']'); if (!sequential) http://git-wip-us.apache.org/repos/asf/ignite/blob/7b6e17b2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java index de18c44..69df748 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java @@ -41,8 +41,8 @@ import org.apache.ignite.internal.IgniteNodeAttributes; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiPredicate; @@ -192,8 +192,6 @@ public class IgniteClientCacheStartFailoverTest extends GridCommonAbstractTest { */ @Test public void testClientStartLastServerFailsMvccTx() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-10262";); - clientStartLastServerFails(TRANSACTIONAL_SNAPSHOT); } http://git-wip-us.apache.org/repos/asf/ignite/blob/7b6e17b2/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index c2f3f89..29f39b9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1140,8 +1140,16 @@ public class IgniteH2Indexing implements GridQueryIndexing { return new Iterable<List<?>>() { @SuppressWarnings("NullableProblems") @Override public Iterator<List<?>> iterator() { - return rdcQryExec.query(schemaName, qry, keepCacheObj, enforceJoinOrder, opTimeout, - cancel, params, parts, lazy, tracker); + try { + return rdcQryExec.query(schemaName, qry, keepCacheObj, enforceJoinOrder, opTimeout, + cancel, params, parts, lazy, tracker); + } + catch (Throwable e) { + if (tracker != null) + tracker.onDone(); + + throw e; + } } }; } http://git-wip-us.apache.org/repos/asf/ignite/blob/7b6e17b2/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java index b14dc07..e986f44 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractSqlCoordinatorFailoverTest.java @@ -17,10 +17,26 @@ package org.apache.ignite.internal.processors.cache.mvcc; +import java.util.concurrent.Callable; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheServerNotFoundException; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteNodeAttributes; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL; import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.DML; @@ -152,4 +168,149 @@ public abstract class CacheMvccAbstractSqlCoordinatorFailoverTest extends CacheM public void testCoordinatorChangeActiveQueryClientFails_SimpleScan() throws Exception { checkCoordinatorChangeActiveQueryClientFails_Simple(new InitIndexing(Integer.class, Integer.class), SCAN, DML); } + + /** + * @throws Exception If failed. + */ + @Test + public void testStartLastServerFails() throws Exception { + testSpi = true; + + startGrids(3); + + CacheConfiguration<Object, Object> cfg = cacheConfiguration(cacheMode(), FULL_SYNC, 0, DFLT_PARTITION_COUNT) + .setIndexedTypes(Integer.class, Integer.class); + + cfg.setNodeFilter(new TestNodeFilter(getTestIgniteInstanceName(1))); + + Ignite srv1 = ignite(1); + + srv1.createCache(cfg); + + client = true; + + final Ignite c = startGrid(3); + + client = false; + + TestRecordingCommunicationSpi.spi(srv1).blockMessages(GridDhtAffinityAssignmentResponse.class, c.name()); + + IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() { + @Override public Void call() throws Exception { + c.cache(DEFAULT_CACHE_NAME); + + return null; + } + }, "start-cache"); + + U.sleep(1000); + + assertFalse(fut.isDone()); + + stopGrid(1); + + fut.get(); + + final IgniteCache<Object, Object> clientCache = c.cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < 10; i++) { + final int k = i; + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.get(k); + + return null; + } + }, CacheServerNotFoundException.class, null); + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.put(k, k); + + return null; + } + }, CacheServerNotFoundException.class, null); + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.remove(k); + + return null; + } + }, CacheServerNotFoundException.class, null); + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("SELECT * FROM INTEGER")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("SELECT * FROM INTEGER ORDER BY _val")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("DELETE FROM Integer WHERE 1 = 1")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("INSERT INTO Integer (_key, _val) VALUES (1, 2)")).getAll(); + + return null; + } + }, CacheException.class, "Failed to get primary node"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + + GridTestUtils.assertThrows(log, new Callable<Void>() { + @Override public Void call() throws Exception { + clientCache.query(new SqlFieldsQuery("UPDATE Integer SET _val=42 WHERE _key IN (SELECT DISTINCT _val FROM INTEGER)")).getAll(); + + return null; + } + }, CacheException.class, "Failed to find data nodes for cache"); // TODO IGNITE-10377 should be CacheServerNotFoundException. + } + + startGrid(1); + + awaitPartitionMapExchange(); + + for (int i = 0; i < 100; i++) { + assertNull(clientCache.get(i)); + + clientCache.put(i, i); + + assertEquals(i, clientCache.get(i)); + } + } + + /** + * + */ + private static class TestNodeFilter implements IgnitePredicate<ClusterNode> { + /** */ + private final String includeName; + + /** + * @param includeName Node to include. + */ + public TestNodeFilter(String includeName) { + this.includeName = includeName; + } + + /** {@inheritDoc} */ + @Override public boolean apply(ClusterNode node) { + return includeName.equals(node.attribute(IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME)); + } + } }
