[ https://issues.apache.org/jira/browse/IGNITE-13313?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17172353#comment-17172353 ]
Grigory Domozhirov commented on IGNITE-13313: --------------------------------------------- load pattern: Launching 1 server + 1 client node sequentially on each of 6 hosts. So, 2 nodes on each virtial machine, 12 nodes total: 6 server + 6 clients. Server nodes performs nothing. Each client node creates and massively populates number of caches. Mostly with putAllAsync (wating for previous call future to complete), but also with some invokeAllAsync and sometimes performs clear. Size of batch - up to 100k records. First of laucnhed clients creates and populates about 200 caches, others - from 1 to 10. > IndexOutOfBoundsException from GridDhtAtomicUpdateRequest on server node > startup > -------------------------------------------------------------------------------- > > Key: IGNITE-13313 > URL: https://issues.apache.org/jira/browse/IGNITE-13313 > Project: Ignite > Issue Type: Bug > Reporter: Grigory Domozhirov > Priority: Major > > Sometimes server node fails on startup with following exception. > GGCE 8.7.21, no persistance, atomic caches > > 2020-07-30 13:39:02,962 [sys-stripe-0-#1|#1] ERROR > o.a.i.i.p.c.GridCacheIoManager - Failed processing message > [senderId=98dc3c18-ea57-4805-9603-b92eb7e62be2, msg=GridD > htAtomicUpdateRequest [keys=ArrayList > [com.moex.esb.blackhole.model.fx.trade.Securities$Key [idHash=4701836, > hash=890931906, secboard=AETS, seccode=GLDRUB_TOD], > *_<skipped>_* > ... and 111883 skipped ...=0,_*<skipped>*_ > prevVals=ArrayList [null, null, null, null, null, null, null, null, null, > null, null, null, null, null, null, null, null, null, null, null, null, null, > null, null, > null, null, null, null, null, null, null, null, null, null, null, null, > null, null, null, null, null, null, null, null, null, null, null, null, null, > null, null, n > ull, null, null, null, null, null, null, null, null, null, null, null, null, > null, null, null, null, null, null, null, null, null, null, null, null, null, > null, nul > l, null, null, null, null, null, null, null, null, null, null, null, null, > null, null, null, null, null, null, null, null, null... and 19 more], > ttls=null, conflict > ExpireTimes=null, nearTtls=null, nearExpireTimes=null, nearKeys=null, > nearVals=null, obsoleteIndexes=null, forceTransformBackups=false, > updateCntrs=GridLongList [id > x=174, > arr=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1, > > 1,2,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,2,2,1,1,1,1,2,1,1,3,1,1,1,1,1,2,1,2,1,1,1,1,2,1,1,1,2,1,2,1,2,1,1,1,2,1,2,2,1,4,1,3,3,1,2,2,2,2,1,3,1,1,1,1,1,2,3,4,1,1,2,1,1,1, > 1,4,1,1,1,2,1,2,1,1,1,1,3,1,1,1]], super=GridDhtAtomicAbstractUpdateRequest > [onRes=false, nearNodeId=ae4abad0-d501-4703-98bf-b5eabd10f159, > nearFutId=147459, flags=k > eepBinary|hasRes]]] > java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119 > at > java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) > ~[na:na] > at > java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) > ~[na:na] > at > java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) > ~[na:na] > at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na] > at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) > ~[ignite > -core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) > ~[ignit > e-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) > ~[ignite-core-8.7.21.jar:8. > 7.21] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) > ~[ignite-core-8.7.21.jar:8.7.2 > 1] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) > ~[ignite-core-8.7.21.jar:8.7.2 > 1] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) > ~[ignite-core-8.7.21.jar:8.7.21] > at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na] > > > 2020-07-30 13:39:02,964 [sys-stripe-0-#1|#1] ERROR ROOT - Critical system > error detected. Will be handled accordingly to configured handler > [hnd=StopNodeOrHaltFailureH > andler [tryStop=false, timeout=0, super=AbstractFailureHandler > [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, > SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], fa > ilureCtx=FailureContext [type=CRITICAL_ERROR, > err=java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length > 119]] > java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119 > at > java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) > ~[na:na] > at > java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) > ~[na:na] > at > java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) > ~[na:na] > at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na] > at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) > ~[ignite > -core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) > ~[ignit > e-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) > ~[ignite-core-8.7.21.jar:8. > 7.21] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) > ~[ignite-core-8.7.21.jar:8.7.2 > 1] > at > org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) > ~[ignite-core-8.7.21.jar:8.7.2 > 1] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) > ~[ignite-core-8.7.21.jar:8.7.21] > at > org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) > ~[ignite-core-8.7.21.jar:8.7.21] > at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na] > !https://owa.moex.com/owa/service.svc/s/GetFileAttachment?id=AQMkADI3MWRlYTcxLTk4NTItNDE1Mi04NzhhLTFlNmZlNGJjODBkZgBGAAADA0VgfW9PI06bbEcYGiv%2FdAcActmJFw3L1U%2ByDGn9%2B1yz%2FAAAAgEPAAAActmJFw3L1U%2ByDGn9%2B1yz%2FAACFYvp3gAAAAESABAAwxeg969M%2Fk2HFngHONZ4zA%3D%3D&X-OWA-CANARY=LvEoq6xN8UeyuWHAH2kxHwDiVHXQJNgIWDr-DG2WSjSwvZaJYt6CkBV-j6eaZGO0K24etDWd7Oo.! -- This message was sent by Atlassian Jira (v8.3.4#803005)