Repository: ignite Updated Branches: refs/heads/ignite-8446 c2426df39 -> fa1a33ec6
IGNITE-8797 Exception during writeCheckpointEntry is passed to failure handler during checkpoint finish Signed-off-by: Andrey Gura <ag...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a61e1a90 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a61e1a90 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a61e1a90 Branch: refs/heads/ignite-8446 Commit: a61e1a90ccf65082bf49d481e5d88de5806ebaee Parents: 774ed77 Author: Aleksey Plekhanov <plehanov.a...@gmail.com> Authored: Mon Jul 30 22:43:37 2018 +0300 Committer: Andrey Gura <ag...@apache.org> Committed: Mon Jul 30 22:43:37 2018 +0300 ---------------------------------------------------------------------- .../GridCacheDatabaseSharedManager.java | 46 ++------- .../IgniteClusterActivateDeactivateTest.java | 2 + .../PageEvictionMultinodeMixedRegionsTest.java | 2 + .../IgnitePdsCorruptedStoreTest.java | 99 +++++++++++++++++++- ...lientAffinityAssignmentWithBaselineTest.java | 4 +- .../IgniteBaselineAbstractFullApiSelfTest.java | 2 + ...nitePdsCacheDestroyDuringCheckpointTest.java | 4 +- .../db/file/IgnitePdsPageReplacementTest.java | 2 + .../junits/common/GridCommonAbstractTest.java | 2 + ...ngBaselineCacheQueryNodeRestartSelfTest.java | 2 + ...eBaselineCacheQueryNodeRestartsSelfTest.java | 2 + .../zk/internal/ZookeeperDiscoverySpiTest.java | 4 +- 12 files changed, 128 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 3c78e55..1203de8 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -3059,11 +3059,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan * */ private void doCheckpoint() { + Checkpoint chp = null; + try { CheckpointMetricsTracker tracker = new CheckpointMetricsTracker(); - Checkpoint chp; - try { chp = markCheckpointBegin(tracker); } @@ -3133,16 +3133,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan } // Wait and check for errors. - try { - doneWriteFut.get(); - } catch (IgniteCheckedException e) { - chp.progress.cpFinishFut.onDone(e); - - // In case of checkpoint writing error node should be invalidated and stopped. - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); - - return; - } + doneWriteFut.get(); // Must re-check shutdown flag here because threads may have skipped some pages. // If so, we should not put finish checkpoint mark. @@ -3175,33 +3166,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan snapshotMgr.afterCheckpointPageWritten(); - try { - destroyedPartitionsCnt = destroyEvictedPartitions(); - } - catch (IgniteCheckedException e) { - chp.progress.cpFinishFut.onDone(e); - - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); - - return; - } + destroyedPartitionsCnt = destroyEvictedPartitions(); // Must mark successful checkpoint only if there are no exceptions or interrupts. success = true; } finally { - if (success) { - try { - markCheckpointEnd(chp); - } - catch (IgniteCheckedException e) { - chp.progress.cpFinishFut.onDone(e); - - cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); - - return; - } - } + if (success) + markCheckpointEnd(chp); } tracker.onEnd(); @@ -3249,8 +3221,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan } } catch (IgniteCheckedException e) { - // TODO-ignite-db how to handle exception? - U.error(log, "Failed to create checkpoint.", e); + if (chp != null) + chp.progress.cpFinishFut.onDone(e); + + cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java index cb2d13b..3b39bfb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -1128,6 +1128,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest * @throws Exception If failed. */ private void stateChangeFailover3(boolean activate) throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-8220";); + testReconnectSpi = true; startNodesAndBlockStatusChange(4, 0, 0, !activate); http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java index 3da47e7..848e835 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeMixedRegionsTest.java @@ -53,6 +53,8 @@ public class PageEvictionMultinodeMixedRegionsTest extends PageEvictionMultinode /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java index ff95f97..fc2a7d6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java @@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.persistence; import java.io.File; import java.io.IOException; +import java.nio.file.OpenOption; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; @@ -39,17 +40,24 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.wal.StorageException; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; -import org.apache.ignite.internal.processors.cache.persistence.file.PersistentStorageIOException; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiClosure; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.WRITE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID; @@ -67,6 +75,9 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest { /** Failure handler. */ private DummyFailureHandler failureHnd; + /** Failing FileIO factory. */ + private FailingFileIOFactory failingFileIOFactory; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { cleanPersistenceDir(); @@ -91,12 +102,15 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest { cfg.setConsistentId(igniteInstanceName); + failingFileIOFactory = new FailingFileIOFactory(); + DataStorageConfiguration memCfg = new DataStorageConfiguration() .setDefaultDataRegionConfiguration( new DataRegionConfiguration() .setMaxSize(100 * 1024 * 1024) .setPersistenceEnabled(true) - ); + ) + .setFileIOFactory(failingFileIOFactory); cfg.setDataStorageConfiguration(memCfg); @@ -351,6 +365,48 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest { } } + + /** + * Test node invalidation due to checkpoint error. + */ + public void testCheckpointFailure() throws Exception { + IgniteEx ignite = startGrid(0); + + failingFileIOFactory.createClosure(new IgniteBiClosure<File, OpenOption[], FileIO>() { + @Override public FileIO apply(File file, OpenOption[] options) { + if (file.getName().indexOf("-END.bin") >= 0) { + FileIO delegate; + + try { + delegate = failingFileIOFactory.delegateFactory().create(file, options); + } + catch (IOException ignore) { + return null; + } + + return new FileIODecorator(delegate) { + @Override public void close() throws IOException { + throw new IOException("Checkpoint failed"); + } + }; + } + + return null; + } + }); + + ignite.cluster().active(true); + + try { + forceCheckpoint(ignite); + } + catch (Exception ignore) { + // No-op. + } + + waitFailure(IOException.class); + } + /** * @param expError Expected error. */ @@ -392,4 +448,43 @@ public class IgnitePdsCorruptedStoreTest extends GridCommonAbstractTest { return true; } } + + /** + * Create File I/O which can fail according to implemented closure. + */ + private static class FailingFileIOFactory implements FileIOFactory { + /** Delegate factory. */ + private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); + + /** Create FileIO closure. */ + private volatile IgniteBiClosure<File, OpenOption[], FileIO> createClo; + + /** {@inheritDoc} */ + @Override public FileIO create(File file) throws IOException { + return create(file, CREATE, READ, WRITE); + } + + /** {@inheritDoc} */ + @Override public FileIO create(File file, OpenOption... openOption) throws IOException { + FileIO fileIO = null; + if (createClo != null) + fileIO = createClo.apply(file, openOption); + + return fileIO != null ? fileIO : delegateFactory.create(file, openOption); + } + + /** + * @param createClo FileIO create closure. + */ + public void createClosure(IgniteBiClosure<File, OpenOption[], FileIO> createClo) { + this.createClo = createClo; + } + + /** + * + */ + public FileIOFactory delegateFactory() { + return delegateFactory; + } + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java index 7071f65..7e9765c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java @@ -156,9 +156,9 @@ public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstract /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - cleanPersistenceDir(); - stopAllGrids(); + + cleanPersistenceDir(); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java index 67d13d6..5199fe9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java @@ -44,6 +44,8 @@ public abstract class IgniteBaselineAbstractFullApiSelfTest extends GridCacheAbs @Override protected void afterTestsStopped() throws Exception { super.afterTestsStopped(); + stopAllGrids(); + cleanPersistenceDir(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java index bb20304..10fc386 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java @@ -85,6 +85,8 @@ public class IgnitePdsCacheDestroyDuringCheckpointTest extends GridCommonAbstrac /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); } @@ -147,4 +149,4 @@ public class IgnitePdsCacheDestroyDuringCheckpointTest extends GridCommonAbstrac } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java index f52b32e..432393e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsPageReplacementTest.java @@ -105,6 +105,8 @@ public class IgnitePdsPageReplacementTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 33eae86..2c5091c 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -1712,6 +1712,8 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest { * */ protected void cleanPersistenceDir() throws Exception { + assertTrue("Grids are not stopped", F.isEmpty(G.allGrids())); + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false)); U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false)); U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false)); http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java index b2a5758..b99745f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java @@ -65,6 +65,8 @@ public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteC /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java index d5a0408..0d631fc 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java @@ -61,6 +61,8 @@ public class IgniteStableBaselineCacheQueryNodeRestartsSelfTest extends IgniteCa /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/a61e1a90/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java index 6925a5e..cc80a72 100644 --- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java @@ -479,9 +479,9 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest { // checkZkNodesCleanup(); } finally { - reset(); - stopAllGrids(); + + reset(); } }
