IGNITE-8879 Fix blinking baseline node sometimes unable to connect to cluster - Fixes #4893.
Signed-off-by: Dmitriy Govorukhin <dmitriy.govoruk...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3f7109ff Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3f7109ff Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3f7109ff Branch: refs/heads/ignite-9720 Commit: 3f7109ff52bdfc3157987b2230216cc982392eff Parents: b1584a8 Author: Ivan Daschinskiy <ivanda...@gmail.com> Authored: Tue Oct 23 12:33:24 2018 +0300 Committer: Dmitriy Govorukhin <dmitriy.govoruk...@gmail.com> Committed: Tue Oct 23 12:33:24 2018 +0300 ---------------------------------------------------------------------- .../GridDhtPartitionsExchangeFuture.java | 19 ++++----- .../GridCacheDatabaseSharedManager.java | 22 ++++------ .../IgniteCacheDatabaseSharedManager.java | 6 ++- .../metastorage/IgniteMetaStorageBasicTest.java | 45 ++++++++++++++++++++ 4 files changed, 66 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/3f7109ff/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 0c2cbe2..63d3635 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -871,16 +871,13 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte * @throws IgniteCheckedException If failed. */ private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedException { - if (isLocalNodeNotInBaseline()) { + boolean baselineNode = isLocalNodeInBaseline(); + + if (!baselineNode) { cctx.exchange().exchangerBlockingSectionBegin(); try { cctx.cache().cleanupCachesDirectories(); - - cctx.database().cleanupCheckpointDirectory(); - - if (cctx.wal() != null) - cctx.wal().cleanupWalDirectories(); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -916,7 +913,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte cctx.exchange().exchangerBlockingSectionBegin(); try { - cctx.database().readCheckpointAndRestoreMemory(startDescs); + cctx.database().readCheckpointAndRestoreMemory(startDescs, !baselineNode); } finally { cctx.exchange().exchangerBlockingSectionEnd(); @@ -950,12 +947,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte } /** - * @return {@code true} if local node is not in baseline and {@code false} otherwise. + * @return {@code true} if local node is in baseline and {@code false} otherwise. */ - private boolean isLocalNodeNotInBaseline() { + private boolean isLocalNodeInBaseline() { BaselineTopology topology = cctx.discovery().discoCache().state().baselineTopology(); - return topology!= null && !topology.consistentIds().contains(cctx.localNode().consistentId()); + return topology != null && topology.consistentIds().contains(cctx.localNode().consistentId()); } /** @@ -1098,7 +1095,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte cctx.exchange().exchangerBlockingSectionBegin(); try { - cctx.database().readCheckpointAndRestoreMemory(startDescs); + cctx.database().readCheckpointAndRestoreMemory(startDescs, false); } finally { cctx.exchange().exchangerBlockingSectionEnd(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f7109ff/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 73d65f4..0f37751 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -645,7 +645,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan checkpointReadLock(); try { - restoreMemory(status, true, storePageMem); + restoreMemory(status, true, storePageMem, false); metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true); @@ -790,7 +790,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** {@inheritDoc} */ @Override public void readCheckpointAndRestoreMemory( - List<DynamicCacheDescriptor> cachesToStart + List<DynamicCacheDescriptor> cachesToStart, + boolean restoreMetastorageOnly ) throws IgniteCheckedException { assert !cctx.localNode().isClient(); @@ -819,7 +820,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME) ); - WALPointer restore = restoreMemory(status); + WALPointer restore = restoreMemory(status, restoreMetastorageOnly, (PageMemoryEx) metaStorage.pageMemory(), true); if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) { throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " + @@ -1958,24 +1959,17 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** * @param status Checkpoint status. - * @throws IgniteCheckedException If failed. - * @throws StorageException In case I/O error occurred during operations with storage. - */ - @Nullable private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException { - return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory()); - } - - /** - * @param status Checkpoint status. * @param metastoreOnly If {@code True} restores Metastorage only. * @param storePageMem Metastore page memory. + * @param finalizeCp If {@code True}, finalizes checkpoint on recovery. * @throws IgniteCheckedException If failed. * @throws StorageException In case I/O error occurred during operations with storage. */ @Nullable private WALPointer restoreMemory( CheckpointStatus status, boolean metastoreOnly, - PageMemoryEx storePageMem + PageMemoryEx storePageMem, + boolean finalizeCp ) throws IgniteCheckedException { assert !metastoreOnly || storePageMem != null; @@ -2135,7 +2129,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan } } - if (metastoreOnly) + if (!finalizeCp) return null; WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3f7109ff/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 28ce085..db6b987 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -649,9 +649,13 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** * @param cachesToStart Started caches. + * @param restoreMetastorageOnly Apply updates only for metastorage. * @throws IgniteCheckedException If failed. */ - public void readCheckpointAndRestoreMemory(List<DynamicCacheDescriptor> cachesToStart) throws IgniteCheckedException { + public void readCheckpointAndRestoreMemory( + List<DynamicCacheDescriptor> cachesToStart, + boolean restoreMetastorageOnly + ) throws IgniteCheckedException { // No-op. } http://git-wip-us.apache.org/repos/asf/ignite/blob/3f7109ff/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java index 1837515..e5a53fa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java @@ -17,12 +17,14 @@ package org.apache.ignite.internal.processors.cache.persistence.metastorage; import java.io.Serializable; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -106,6 +108,34 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest { verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX); } + /** + * @throws Exception If fails. + */ + public void testRecoveryOfMetastorageWhenNodeNotInBaseline() throws Exception { + IgniteEx ig0 = startGrid(0); + + ig0.cluster().active(true); + + final byte KEYS_CNT = 100; + final String KEY_PREFIX = "test.key."; + final String NEW_VAL_PREFIX = "new.val."; + final String UPDATED_VAL_PREFIX = "updated.val."; + + startGrid(1); + + // Disable checkpoints in order to check whether recovery works. + forceCheckpoint(grid(1)); + disableCheckpoints(grid(1)); + + loadKeys(grid(1), KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX); + + stopGrid(1, true); + + startGrid(1); + + verifyKeys(grid(1), KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX); + } + /** */ private void loadKeys(IgniteEx ig, byte keysCnt, @@ -144,4 +174,19 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest { Assert.assertEquals(valPrefix + i, val); } } + + /** + * Disable checkpoints on a specific node. + * + * @param node Ignite node.h + * @throws IgniteCheckedException If failed. + */ + private void disableCheckpoints(Ignite node) throws IgniteCheckedException { + assert !node.cluster().localNode().isClient(); + + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)node).context() + .cache().context().database(); + + dbMgr.enableCheckpoints(false).get(); + } }