IGNITE-8069 IgniteOutOfMemoryException should be handled accordingly to provided failure handler
Signed-off-by: Andrey Gura <ag...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9bb4ce8a Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9bb4ce8a Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9bb4ce8a Branch: refs/heads/ignite-8201 Commit: 9bb4ce8ab9770967d39f0acac5cdc1dc4230abb4 Parents: c807ae9 Author: Aleksey Plekhanov <plehanov.a...@gmail.com> Authored: Tue Apr 10 18:54:03 2018 +0300 Committer: Andrey Gura <ag...@apache.org> Committed: Tue Apr 10 18:54:03 2018 +0300 ---------------------------------------------------------------------- .../pagemem/impl/PageMemoryNoStoreImpl.java | 17 ++- .../persistence/pagemem/PageMemoryImpl.java | 17 ++- .../failure/AbstractFailureHandlerTest.java | 74 ++++++++++ .../ignite/failure/IoomFailureHandlerTest.java | 144 +++++++++++++++++++ .../persistence/pagemem/PageMemoryImplTest.java | 9 ++ .../ignite/testsuites/IgniteBasicTestSuite.java | 2 + 6 files changed, 259 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java index 7424af6..d4b22a6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java @@ -28,6 +28,8 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; @@ -158,6 +160,9 @@ public class PageMemoryNoStoreImpl implements PageMemory { /** */ private final boolean trackAcquiredPages; + /** Shared context. */ + private final GridCacheSharedContext<?, ?> ctx; + /** * @param log Logger. * @param directMemoryProvider Memory allocator to use. @@ -184,6 +189,7 @@ public class PageMemoryNoStoreImpl implements PageMemory { this.trackAcquiredPages = trackAcquiredPages; this.memMetrics = memMetrics; this.dataRegionCfg = dataRegionCfg; + this.ctx = sharedCtx; sysPageSize = pageSize + PAGE_OVERHEAD; @@ -288,8 +294,8 @@ public class PageMemoryNoStoreImpl implements PageMemory { } } - if (relPtr == INVALID_REL_PTR) - throw new IgniteOutOfMemoryException("Out of memory in data region [" + + if (relPtr == INVALID_REL_PTR) { + IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" + "name=" + dataRegionCfg.getName() + ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) + ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) + @@ -299,6 +305,13 @@ public class PageMemoryNoStoreImpl implements PageMemory { " ^-- Enable eviction or expiration policies" ); + if (ctx != null) + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom)); + + throw oom; + } + + assert (relPtr & ~PageIdUtils.PAGE_IDX_MASK) == 0 : U.hexLong(relPtr & ~PageIdUtils.PAGE_IDX_MASK); // Assign page ID according to flags and partition ID. http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 46fb7dd..4463224 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -40,6 +40,8 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; @@ -543,7 +545,7 @@ public class PageMemoryImpl implements PageMemoryEx { catch (IgniteOutOfMemoryException oom) { DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration(); - throw (IgniteOutOfMemoryException) new IgniteOutOfMemoryException("Out of memory in data region [" + + IgniteOutOfMemoryException e = new IgniteOutOfMemoryException("Out of memory in data region [" + "name=" + dataRegionCfg.getName() + ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) + ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) + @@ -551,7 +553,13 @@ public class PageMemoryImpl implements PageMemoryEx { " ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() + " ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() + " ^-- Enable eviction or expiration policies" - ).initCause(oom); + ); + + e.initCause(oom); + + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + + throw e; } finally { seg.writeLock().unlock(); @@ -746,6 +754,11 @@ public class PageMemoryImpl implements PageMemoryEx { return absPtr; } + catch (IgniteOutOfMemoryException oom) { + ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom)); + + throw oom; + } finally { seg.writeLock().unlock(); http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java new file mode 100644 index 0000000..dc5f1f5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.failure; + +import org.apache.ignite.Ignite; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Abstract failure handler test. + */ +public class AbstractFailureHandlerTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected FailureHandler getFailureHandler(String igniteInstanceName) { + return new DummyFailureHandler(); + } + + /** + * Gets dummy failure handler for ignite instance. + * + * @param ignite Ignite. + */ + protected static DummyFailureHandler dummyFailureHandler(Ignite ignite) { + return (DummyFailureHandler)ignite.configuration().getFailureHandler(); + } + + /** + * + */ + protected static class DummyFailureHandler implements FailureHandler { + /** Failure. */ + private volatile boolean failure; + + /** Failure context. */ + private volatile FailureContext ctx; + + /** {@inheritDoc} */ + @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) { + failure = true; + + ctx = failureCtx; + + return true; + } + + /** + * @return Failure. + */ + public boolean failure() { + return failure; + } + + /** + * @return Failure context. + */ + public FailureContext failureContext() { + return ctx; + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java new file mode 100644 index 0000000..a777f81 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.failure; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.transactions.Transaction; + +/** + * IgniteOutOfMemoryError failure handler test. + */ +public class IoomFailureHandlerTest extends AbstractFailureHandlerTest { + /** Offheap size for memory policy. */ + private static final int SIZE = 10 * 1024 * 1024; + + /** Page size. */ + static final int PAGE_SIZE = 2048; + + /** Number of entries. */ + static final int ENTRIES = 10_000; + + /** PDS enabled. */ + private boolean pds; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration(); + + DataRegionConfiguration dfltPlcCfg = new DataRegionConfiguration(); + + dfltPlcCfg.setName("dfltPlc"); + dfltPlcCfg.setInitialSize(SIZE); + dfltPlcCfg.setMaxSize(SIZE); + + if (pds) + dfltPlcCfg.setPersistenceEnabled(true); + + dsCfg.setDefaultDataRegionConfiguration(dfltPlcCfg); + dsCfg.setPageSize(PAGE_SIZE); + + cfg.setDataStorageConfiguration(dsCfg); + + CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>() + .setName(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.PARTITIONED) + .setBackups(0) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + cleanPersistenceDir(); + + super.beforeTest(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanPersistenceDir(); + } + + /** + * Test IgniteOutOfMemoryException handling with no store. + */ + public void testIoomErrorNoStoreHandling() throws Exception { + testIoomErrorHandling(false); + } + + /** + * Test IgniteOutOfMemoryException handling with PDS. + */ + public void testIoomErrorPdsHandling() throws Exception { + testIoomErrorHandling(true); + } + + /** + * Test IOOME handling. + */ + public void testIoomErrorHandling(boolean pds) throws Exception { + this.pds = pds; + + IgniteEx ignite0 = startGrid(0); + IgniteEx ignite1 = startGrid(1); + + try { + if (pds) + ignite0.cluster().active(true); + + IgniteCache<Integer, Object> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME); + IgniteCache<Integer, Object> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + try (Transaction tx = ignite0.transactions().txStart()) { + for (Integer i : primaryKeys(cache1, ENTRIES)) + cache0.put(i, new byte[PAGE_SIZE / 3 * 2]); + + tx.commit(); + } + catch (Throwable ignore) { + // Expected. + } + + assertFalse(dummyFailureHandler(ignite0).failure()); + assertTrue(dummyFailureHandler(ignite1).failure()); + assertTrue(X.hasCause(dummyFailureHandler(ignite1).failureContext().error(), IgniteOutOfMemoryException.class)); + } + finally { + stopGrid(1); + stopGrid(0); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java index 31af118..3697c4c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.NoOpFailureHandler; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; @@ -41,6 +42,7 @@ import org.apache.ignite.internal.processors.cache.persistence.CheckpointWritePr import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.failure.FailureProcessor; import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; import org.apache.ignite.internal.util.lang.GridInClosure3X; import org.apache.ignite.plugin.PluginProvider; @@ -268,10 +270,17 @@ public class PageMemoryImplTest extends GridCommonAbstractTest { IgniteConfiguration igniteCfg = new IgniteConfiguration(); igniteCfg.setDataStorageConfiguration(new DataStorageConfiguration()); + igniteCfg.setFailureHandler(new NoOpFailureHandler()); GridTestKernalContext kernalCtx = new GridTestKernalContext(new GridTestLog4jLogger(), igniteCfg); kernalCtx.add(new IgnitePluginProcessor(kernalCtx, igniteCfg, Collections.<PluginProvider>emptyList())); + FailureProcessor failureProc = new FailureProcessor(kernalCtx); + + failureProc.start(); + + kernalCtx.add(failureProc); + GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>( kernalCtx, null, http://git-wip-us.apache.org/repos/asf/ignite/blob/9bb4ce8a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index dd9cdfd..c4b7d92 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -21,6 +21,7 @@ import java.util.Set; import junit.framework.TestSuite; import org.apache.ignite.GridSuppressedExceptionSelfTest; import org.apache.ignite.failure.FailureHandlerTriggeredTest; +import org.apache.ignite.failure.IoomFailureHandlerTest; import org.apache.ignite.failure.StopNodeFailureHandlerTest; import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest; import org.apache.ignite.internal.ClassSetTest; @@ -197,6 +198,7 @@ public class IgniteBasicTestSuite extends TestSuite { suite.addTestSuite(FailureHandlerTriggeredTest.class); suite.addTestSuite(StopNodeFailureHandlerTest.class); suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class); + suite.addTestSuite(IoomFailureHandlerTest.class); return suite; }