Denis Chudov created IGNITE-13418: ------------------------------------- Summary: Deadlock on multiple cache delete Key: IGNITE-13418 URL: https://issues.apache.org/jira/browse/IGNITE-13418 Project: Ignite Issue Type: Bug Reporter: Denis Chudov Assignee: Denis Chudov
Thread 1: - acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone - acquires GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop - enters H2TreeIndex.destroy - releases checkpoint read lock in H2Tree.temporaryReleaseLock, can't take it again because of db-checkpoint-thread Thread 2: - acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone - trying to acquire GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop which is held by thread 1 db-checkpoint-thread: - trying to acquire checkpoint write lock, can't do it because of Thread 2 Decision: H2Tree.temporaryReleaseLock should release lock only in case when tree deletion is asynchronous (H2TreeIndex.destroy is called with async=true), i.e. it happens inside of DurableBackgroundTask. Such tasks are executed in separate threads, which don't hold any other locks. Thread dump: {code:java} Thread [name="sys-#1220%DPL_GRID%DplGridNodeName%", id=3200, state=BLOCKED, blockCnt=1, waitCnt=0] Lock [object=java.lang.Object@6a9a92ba, ownerName=sys-#1215%DPL_GRID%DplGridNodeName%, ownerId=3195] at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1695) at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902) at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169) at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644) at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803) at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source) at o.a.i.i.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10879) at o.a.i.i.util.IgniteUtils$$Lambda$436/321848940.call(Unknown Source) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked synchronizers: java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2582f93c java.util.concurrent.ThreadPoolExecutor$Worker@4da1cafe Thread [name="sys-#1215%DPL_GRID%DplGridNodeName%", id=3195, state=BLOCKED, blockCnt=4, waitCnt=437520] Lock [object=o.a.i.i.processors.failure.FailureProcessor@78edb1e9, ownerName=async-durable-background-task-executor-1-#1222%DPL_GRID%DplGridNodeName%, ownerId=3202] at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:162) at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:151) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.failCheckpointReadLock(GridCacheDatabaseSharedManager.java:1787) at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.checkpointReadLock(GridCacheDatabaseSharedManager.java:1722) at o.a.i.i.processors.query.h2.database.H2Tree.temporaryReleaseLock(H2Tree.java:690) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.temporaryReleaseLock(BPlusTree.java:2367) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2548) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2441) at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2392) at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy0(H2TreeIndex.java:671) at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy(H2TreeIndex.java:639) at o.a.i.i.processors.query.h2.opt.GridH2Table.destroy(GridH2Table.java:567) at o.a.i.i.processors.query.h2.H2TableDescriptor.onDrop(H2TableDescriptor.java:347) at o.a.i.i.processors.query.h2.H2Schema.drop(H2Schema.java:127) at o.a.i.i.processors.query.h2.IgniteH2Indexing.unregisterCache(IgniteH2Indexing.java:2595) at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1727) - locked java.lang.Object@6a9a92ba at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902) at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169) at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644) at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803) at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source) at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10898) at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10800) at o.a.i.i.processors.cache.GridCacheProcessor.processCacheStopRequestOnExchangeDone(GridCacheProcessor.java:2775) at o.a.i.i.processors.cache.GridCacheProcessor.onExchangeDone(GridCacheProcessor.java:2899) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2210) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processFullMessage(GridDhtPartitionsExchangeFuture.java:4197) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$1600(GridDhtPartitionsExchangeFuture.java:143) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3879) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3867) at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:385) at o.a.i.i.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:355) at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveFullMessage(GridDhtPartitionsExchangeFuture.java:3867) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1798) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:408) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:395) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3492) at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3471) at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1106) at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:589) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:386) at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:312) at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:102) at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:301) at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1574) at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1198) at o.a.i.i.managers.communication.GridIoManager.access$4300(GridIoManager.java:132) at o.a.i.i.managers.communication.GridIoManager$8.run(GridIoManager.java:1099) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748){code} -- This message was sent by Atlassian Jira (v8.3.4#803005)