IGNITE-8021 Delete cache config files when cache is destroyed - Fixes #3697.
Signed-off-by: Alexey Goncharuk <alexey.goncha...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2edcb22f Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2edcb22f Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2edcb22f Branch: refs/heads/ignite-7708 Commit: 2edcb22fbb566981097733af6470ed6dde8e786b Parents: 1b3a292 Author: Ivan Daschinskiy <ivanda...@gmail.com> Authored: Tue Apr 17 18:05:42 2018 +0300 Committer: Alexey Goncharuk <alexey.goncha...@gmail.com> Committed: Tue Apr 17 18:05:42 2018 +0300 ---------------------------------------------------------------------- .../pagemem/store/IgnitePageStoreManager.java | 9 + .../processors/cache/GridCacheProcessor.java | 11 + .../persistence/file/FilePageStoreManager.java | 47 +++ ...eConfigurationDataAfterDestroyCacheTest.java | 326 +++++++++++++++++++ .../pagemem/NoOpPageStoreManager.java | 5 + .../ignite/testsuites/IgnitePdsTestSuite.java | 2 + 6 files changed, 400 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java index 1b46bf9..0fc9f94 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java @@ -193,6 +193,15 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh * @throws IgniteCheckedException If failed. */ public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException; + + /** + * Remove cache configuration data file. + * + * @param cacheData Cache configuration. + * @throws IgniteCheckedException If failed. + */ + public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException; + /** * @param grpId Cache group ID. * @return {@code True} if index store for given cache group existed before node started. http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 36edd72..bceb8c7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -1284,6 +1284,17 @@ public class GridCacheProcessor extends GridProcessorAdapter { U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore())); + IgnitePageStoreManager pageStore; + + if (destroy && (pageStore = sharedCtx.pageStore()) != null) { + try { + pageStore.removeCacheData(new StoredCacheData(ctx.config())); + } catch (IgniteCheckedException e) { + U.error(log, "Failed to delete cache configuration data while destroying cache" + + "[cache=" + ctx.name() + "]", e); + } + } + if (log.isInfoEnabled()) { if (ctx.group().sharedGroup()) log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']'); http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 6313eac..837f3d0 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -281,6 +282,9 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null); + if (destroy) + removeCacheGroupConfigurationData(grp); + if (ex != null) throw ex; } @@ -746,6 +750,49 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen } /** + * Delete caches' configuration data files of cache group. + * + * @param ctx Cache group context. + * @throws IgniteCheckedException If fails. + */ + private void removeCacheGroupConfigurationData(CacheGroupContext ctx) throws IgniteCheckedException { + File cacheGrpDir = cacheWorkDir(ctx.sharedGroup(), ctx.cacheOrGroupName()); + + if (cacheGrpDir != null && cacheGrpDir.exists()) { + DirectoryStream.Filter<Path> cacheCfgFileFilter = new DirectoryStream.Filter<Path>() { + @Override public boolean accept(Path path) { + return Files.isRegularFile(path) && path.getFileName().toString().endsWith(CACHE_DATA_FILENAME); + } + }; + + try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(cacheGrpDir.toPath(), cacheCfgFileFilter)) { + for(Path path: dirStream) + Files.deleteIfExists(path); + } + catch (IOException e) { + throw new IgniteCheckedException("Failed to delete cache configurations of group: " + ctx.toString(), e); + } + } + } + + /** {@inheritDoc} */ + @Override public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException { + CacheConfiguration cacheCfg = cacheData.config(); + File cacheWorkDir = cacheWorkDir(cacheCfg); + File file; + + if (cacheData.config().getGroupName() != null) + file = new File(cacheWorkDir, cacheCfg.getName() + CACHE_DATA_FILENAME); + else + file = new File(cacheWorkDir, CACHE_DATA_FILENAME); + + if (file.exists()) { + if (!file.delete()) + throw new IgniteCheckedException("Failed to delete cache configuration:" + cacheCfg.getName()); + } + } + + /** * @param store Store to shutdown. * @param cleanFile {@code True} if files should be cleaned. * @param aggr Aggregating exception. http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java new file mode 100644 index 0000000..d2767d4 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence; + +import java.util.ArrayList; +import java.util.List; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Test correct clean up cache configuration data after destroying cache. + */ +public class IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static final int CACHES = 3; + + /** */ + private static final int NODES = 3; + + /** */ + private static final int NUM_OF_KEYS = 100; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + return cfg.setDiscoverySpi(new TcpDiscoverySpi() + .setIpFinder(IP_FINDER)) + .setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration() + .setMaxSize(200 * 1024 * 1024) + .setPersistenceEnabled(true))); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** + * {@inheritDoc} + * @returns always {@code true} in order to be able to kill nodes when checkpointer thread hangs. + */ + @Override protected boolean isMultiJvm() { + return true; + } + + /** + * Test destroy non grouped caches. + * + * @throws Exception If failed. + */ + public void testDestroyCaches() throws Exception { + Ignite ignite = startGrids(NODES); + + ignite.cluster().active(true); + + startCachesDynamically(ignite); + + checkDestroyCaches(ignite); + } + + /** + * Test destroy grouped caches. + * + * @throws Exception If failed. + */ + public void testDestroyGroupCaches() throws Exception { + Ignite ignite = startGrids(NODES); + + ignite.cluster().active(true); + + startGroupCachesDynamically(ignite); + + checkDestroyCaches(ignite); + } + + /** + * Test destroy caches with disabled checkpoints. + * + * @throws Exception If failed. + */ + public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception { + Ignite ignite = startGrids(NODES); + + ignite.cluster().active(true); + + startCachesDynamically(ignite); + + enableCheckpoints(false); + + checkDestroyCachesAbruptly(ignite); + } + + /** + * Test destroy group caches with disabled checkpoints. + * + * @throws Exception If failed. + */ + public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception { + Ignite ignite = startGrids(NODES); + + ignite.cluster().active(true); + + startGroupCachesDynamically(ignite); + + enableCheckpoints(false); + + checkDestroyCachesAbruptly(ignite); + } + + /** + * Test destroy caches abruptly with checkpoints. + * + * @throws Exception If failed. + */ + public void testDestroyCachesAbruptly() throws Exception { + Ignite ignite = startGrids(NODES); + + ignite.cluster().active(true); + + startCachesDynamically(ignite); + + checkDestroyCachesAbruptly(ignite); + } + + + /** + * Test destroy group caches abruptly with checkpoints. + * + * @throws Exception If failed. + */ + public void testDestroyGroupCachesAbruptly() throws Exception { + Ignite ignite = startGrids(NODES); + + ignite.cluster().active(true); + + startGroupCachesDynamically(ignite); + + checkDestroyCachesAbruptly(ignite); + } + + /** + * @param ignite Ignite. + */ + private void loadCaches(Ignite ignite) { + for (int i = 0; i < CACHES; i++) { + try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cacheName(i))) { + s.allowOverwrite(true); + + for (int j = 0; j < NUM_OF_KEYS; j++) + s.addData(j, "cache: " + i + " data: " + j); + + s.flush(); + } + } + } + + /** + * @param ignite Ignite. + */ + private void checkDestroyCaches(Ignite ignite) throws Exception { + loadCaches(ignite); + + log.warning("destroying caches...."); + + ignite.cache(cacheName(0)).destroy(); + ignite.cache(cacheName(1)).destroy(); + + assertEquals(CACHES - 2, ignite.cacheNames().size()); + + log.warning("Stopping grid"); + + stopAllGrids(); + + log.warning("Grid stopped"); + + log.warning("Starting grid"); + + ignite = startGrids(NODES); + + log.warning("Grid started"); + + assertEquals("Check that caches don't survived", CACHES - 2, ignite.cacheNames().size()); + + for(Ignite ig: G.allGrids()) { + IgniteCache cache = ig.cache(cacheName(2)); + + for (int j = 0; j < NUM_OF_KEYS; j++) + assertNotNull("Check that cache2 contains key: " + j + " node: " + ignite.name(), cache.get(j)); + } + } + + + /** + * @param ignite Ignite instance. + */ + private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception { + loadCaches(ignite); + + log.warning("Destroying caches"); + + ((GatewayProtectedCacheProxy)ignite.cache(cacheName(0))).destroyAsync(); + ((GatewayProtectedCacheProxy)ignite.cache(cacheName(1))).destroyAsync(); + + log.warning("Stopping grid"); + + stopAllGrids(); + + log.warning("Grid stopped"); + + log.warning("Starting grid"); + + startGrids(NODES); + + log.warning("Grid started"); + + for(Ignite ig: G.allGrids()) { + assertTrue(ig.cacheNames().contains(cacheName(2))); + + IgniteCache cache = ig.cache(cacheName(2)); + + for (int j = 0; j < NUM_OF_KEYS; j++) + assertNotNull("Check that survived cache cache2 contains key: " + j + " node: " + ig.name(), cache.get(j)); + } + } + + /** + * @param ignite Ignite. + */ + private void startCachesDynamically(Ignite ignite) { + List<CacheConfiguration> ccfg = new ArrayList<>(CACHES); + + for (int i = 0; i < CACHES; i++) + ccfg.add(new CacheConfiguration<>(cacheName(i)) + .setBackups(1) + .setAffinity(new RendezvousAffinityFunction(false, 32))); + + ignite.createCaches(ccfg); + } + + /** + * @param ignite Ignite instance. + */ + private void startGroupCachesDynamically(Ignite ignite) { + List<CacheConfiguration> ccfg = new ArrayList<>(CACHES); + + for (int i = 0; i < CACHES; i++) + ccfg.add(new CacheConfiguration<>(cacheName(i)) + .setGroupName(i % 2 == 0 ? "grp-even" : "grp-odd") + .setBackups(1) + .setAffinity(new RendezvousAffinityFunction(false, 32))); + + ignite.createCaches(ccfg); + } + + + /** + * Generate cache name from idx. + * + * @param idx Index. + */ + private String cacheName(int idx) { + return "cache" + idx; + } + + /** + * Enable/disable checkpoints on multi JVM nodes only. + * + * @param enabled Enabled flag. + * @throws IgniteCheckedException If failed. + */ + private void enableCheckpoints(boolean enabled) throws IgniteCheckedException { + for (Ignite ignite : G.allGrids()) { + assert !ignite.cluster().localNode().isClient(); + + GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context() + .cache().context().database(); + + dbMgr.enableCheckpoints(enabled).get(); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java index 64acf02..be40c90 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java @@ -187,6 +187,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ + @Override public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException { + // No-op. + } + + /** {@inheritDoc} */ @Override public boolean hasIndexStore(int grpId) { return false; } http://git-wip-us.apache.org/repos/asf/ignite/blob/2edcb22f/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java index a9668e7..af0b7ad 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java @@ -19,6 +19,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence; +import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest; @@ -113,6 +114,7 @@ public class IgnitePdsTestSuite extends TestSuite { suite.addTestSuite(IgnitePdsCacheRestoreTest.class); suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class); + suite.addTestSuite(IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.class); suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);