I'm running compute tasks to load csv files into an a cache. Program runs
fine on one node but I get out of memory errors (Java heap) when running on
3 vms. It's 2.2 and I've tried various config options. If I comment out the
stream use then Ignite still seems to stop after several 1000 tasks which
may be a secondary issue. I've tried various config options (setting max
message queue size and FULL_SYNC on the cache).
SEVERE: Failed to map task jobs to nodes due to undeclared user exception
[cause=Java heap space, ses=GridTaskSessionImpl
[taskName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
dep=LocalDeployment [super=GridDeployment [ts=1509377011339, depMode=SHARED,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, userVer=0,
loc=true, sampleClsName=java.lang.String, pendingUndeploy=false,
undeployed=false, usage=0]],
taskClsName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
sesId=b0591ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
startTime=1509377606896, endTime=9223372036854775807,
taskNodeId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752, closed=false, cpSpi=null,
failSpi=null, loadSpi=null, usage=1, fullSup=false, internal=false,
subjId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0, mapFut=IgniteFuture
[orig=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=null,
hash=1266438780]], execName=null]]
java.lang.OutOfMemoryError: Java heap space
at
org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream.arrayCopy(BinaryHeapOutputStream.java:76)
at
org.apache.ignite.internal.binary.BinaryWriterExImpl.array(BinaryWriterExImpl.java:213)
at
org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:250)
at
org.apache.ignite.internal.binary.BinaryMarshaller.marshal0(BinaryMarshaller.java:82)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.marshal(AbstractNodeNameAwareMarshaller.java:58)
at
org.apache.ignite.internal.util.IgniteUtils.marshal(IgniteUtils.java:9815)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.sendRequest(GridTaskWorker.java:1353)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.processMappedJobs(GridTaskWorker.java:640)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.body(GridTaskWorker.java:532)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.startTask(GridTaskProcessor.java:749)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.execute(GridTaskProcessor.java:448)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:673)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:478)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:464)
at
org.apache.ignite.internal.executor.GridExecutorService.submit(GridExecutorService.java:263)
at com.geospock.ignited.Server.start(Server.kt:210)
at com.geospock.ignited.App.main(App.kt:11)
Oct 30, 2017 3:33:27 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Runtime error caught during grid runnable execution: GridTaskWorker
[internal=false, marsh=o.a.i.i.binary.BinaryMarshaller@63372b46,
ses=GridTaskSessionImpl
[taskName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
dep=LocalDeployment [super=GridDeployment [ts=1509377011339, depMode=SHARED,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, userVer=0,
loc=true, sampleClsName=java.lang.String, pendingUndeploy=false,
undeployed=false, usage=0]],
taskClsName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
sesId=b0591ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
startTime=1509377606896, endTime=9223372036854775807,
taskNodeId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752, closed=false, cpSpi=null,
failSpi=null, loadSpi=null, usage=1, fullSup=false, internal=false,
subjId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0, mapFut=IgniteFuture
[orig=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=null,
hash=1266438780]], execName=null], fut=ComputeTaskInternalFuture
[ses=GridTaskSessionImpl
[taskName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
dep=LocalDeployment [super=GridDeployment [ts=1509377011339, depMode=SHARED,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, userVer=0,
loc=true, sampleClsName=java.lang.String, pendingUndeploy=false,
undeployed=false, usage=0]],
taskClsName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
sesId=b0591ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
startTime=1509377606896, endTime=9223372036854775807,
taskNodeId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752, closed=false, cpSpi=null,
failSpi=null, loadSpi=null, usage=1, fullSup=false, internal=false,
subjId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0, mapFut=IgniteFuture
[orig=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=null,
hash=1266438780]], execName=null], super=GridFutureAdapter
[ignoreInterrupts=false, state=DONE, res=class
o.a.i.compute.ComputeUserUndeclaredException: Failed to map task jobs to
nodes due to undeclared user exception [cause=Java heap space,
ses=GridTaskSessionImpl
[taskName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
dep=LocalDeployment [super=GridDeployment [ts=1509377011339, depMode=SHARED,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, userVer=0,
loc=true, sampleClsName=java.lang.String, pendingUndeploy=false,
undeployed=false, usage=0]],
taskClsName=com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
sesId=b0591ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
startTime=1509377606896, endTime=9223372036854775807,
taskNodeId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752, closed=false, cpSpi=null,
failSpi=null, loadSpi=null, usage=1, fullSup=false, internal=false,
subjId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0, mapFut=IgniteFuture
[orig=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=null,
hash=1266438780]], execName=null]], hash=278071384]], state=FINISHING,
dep=LocalDeployment [super=GridDeployment [ts=1509377011339, depMode=SHARED,
clsLdr=sun.misc.Launcher$AppClassLoader@33909752,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, userVer=0,
loc=true, sampleClsName=java.lang.String, pendingUndeploy=false,
undeployed=false, usage=0]], taskCls=class
com.geospock.ignited.Server$start$$inlined$mapIndexed$lambda$1,
task=o.a.i.i.processors.closure.GridClosureProcessor$T7@b27c911,
continuous=false, lockRespProc=true, resCache=true, noFailover=false,
affPartId=-1, affCacheName=null, mapTopVer=null, retryAttemptCnt=0,
subjId=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
mapper=o.a.i.i.processors.task.GridTaskWorker$1@5110ae28]
java.lang.OutOfMemoryError: Java heap space
at
org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream.arrayCopy(BinaryHeapOutputStream.java:76)
at
org.apache.ignite.internal.binary.BinaryWriterExImpl.array(BinaryWriterExImpl.java:213)
at
org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:250)
at
org.apache.ignite.internal.binary.BinaryMarshaller.marshal0(BinaryMarshaller.java:82)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.marshal(AbstractNodeNameAwareMarshaller.java:58)
at
org.apache.ignite.internal.util.IgniteUtils.marshal(IgniteUtils.java:9815)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.sendRequest(GridTaskWorker.java:1353)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.processMappedJobs(GridTaskWorker.java:640)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.body(GridTaskWorker.java:532)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.startTask(GridTaskProcessor.java:749)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.execute(GridTaskProcessor.java:448)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:673)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:478)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:464)
at
org.apache.ignite.internal.executor.GridExecutorService.submit(GridExecutorService.java:263)
at com.geospock.ignited.Server.start(Server.kt:210)
at com.geospock.ignited.App.main(App.kt:11)
Exception in thread "main" java.lang.OutOfMemoryError: Java heap space
at
org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream.arrayCopy(BinaryHeapOutputStream.java:76)
at
org.apache.ignite.internal.binary.BinaryWriterExImpl.array(BinaryWriterExImpl.java:213)
at
org.apache.ignite.internal.binary.GridBinaryMarshaller.marshal(GridBinaryMarshaller.java:250)
at
org.apache.ignite.internal.binary.BinaryMarshaller.marshal0(BinaryMarshaller.java:82)
at
org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller.marshal(AbstractNodeNameAwareMarshaller.java:58)
at
org.apache.ignite.internal.util.IgniteUtils.marshal(IgniteUtils.java:9815)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.sendRequest(GridTaskWorker.java:1353)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.processMappedJobs(GridTaskWorker.java:640)
at
org.apache.ignite.internal.processors.task.GridTaskWorker.body(GridTaskWorker.java:532)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.startTask(GridTaskProcessor.java:749)
at
org.apache.ignite.internal.processors.task.GridTaskProcessor.execute(GridTaskProcessor.java:448)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:673)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:478)
at
org.apache.ignite.internal.processors.closure.GridClosureProcessor.callAsync(GridClosureProcessor.java:464)
at
org.apache.ignite.internal.executor.GridExecutorService.submit(GridExecutorService.java:263)
at com.geospock.ignited.Server.start(Server.kt:210)
at com.geospock.ignited.App.main(App.kt:11)
remote node:
Oct 30, 2017 3:31:06 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Failed to read message [msg=GridIoMessage [plc=0, topic=null,
topicOrd=-1, ordered=false, timeout=0, skipOnTimeout=false, msg=null],
buf=java.nio.DirectByteBuffer[pos=53 lim=17898 cap=32768],
reader=DirectMessageReader [state=DirectMessageState [pos=0,
stack=[StateItem [stream=DirectByteBufferStreamImplV2
[baseOff=140456302252064, arrOff=-1, tmpArrOff=0, tmpArrBytes=0,
msgTypeDone=true, msg=GridJobExecuteRequest [subjId=null, sesId=null,
jobId=null, startTaskTime=0, timeout=0, taskName=null, userVer=null,
taskClsName=null, ldrParticipants=null, cpSpi=null,
createTime=1509377465329,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, depMode=SHARED,
dynamicSiblings=false, forceLocDep=true, sesFullSup=false, internal=false,
part=0, topVer=null, execName=null], mapIt=null, it=null, arrPos=-1,
keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0,
uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0],
StateItem [stream=DirectByteBufferStreamImplV2 [baseOff=140456302252064,
arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null,
mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0,
prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0,
lastFinished=true], state=0], StateItem [stream=DirectByteBufferStreamImplV2
[baseOff=140456302252064, arrOff=-1, tmpArrOff=0, tmpArrBytes=0,
msgTypeDone=false, msg=null, mapIt=null, it=null, arrPos=-1, keyDone=false,
readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0, uuidMost=0,
uuidLeast=0, uuidLocId=0, lastFinished=true], state=0], StateItem
[stream=DirectByteBufferStreamImplV2 [baseOff=140456302252064, arrOff=-1,
tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null, mapIt=null,
it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0, prim=0,
primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0,
lastFinished=true], state=0], null, null, null, null, null, null]],
lastRead=true], ses=GridSelectorNioSessionImpl [worker=DirectNioClientWorker
[super=AbstractNioClientWorker [idx=1, bytesRcvd=10314443201,
bytesSent=41940044, bytesRcvd0=101826439, bytesSent0=219408, select=true,
super=GridWorker [name=grid-nio-worker-tcp-comm-1, igniteInstanceName=null,
finished=false, hashCode=1221266797, interrupted=false,
runner=grid-nio-worker-tcp-comm-1-#18%null%]]],
writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768],
readBuf=java.nio.DirectByteBuffer[pos=53 lim=17898 cap=32768],
inRecovery=GridNioRecoveryDescriptor [acked=2400, resendCnt=0, rcvCnt=3138,
sentCnt=2410, reserved=true, lastAck=3136, nodeLeft=false,
node=TcpDiscoveryNode [id=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.31.70.152],
sockAddrs=[ip-172-31-70-152.ec2.internal/172.31.70.152:47500,
/0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=3,
intOrder=3, lastExchangeTime=1509377011758, loc=false,
ver=2.2.0#20170915-sha1:5747ce6b, isClient=false], connected=true,
connectCnt=0, queueLimit=4096, reserveCnt=1, pairedConnections=false],
outRecovery=GridNioRecoveryDescriptor [acked=2400, resendCnt=0, rcvCnt=3138,
sentCnt=2410, reserved=true, lastAck=3136, nodeLeft=false,
node=TcpDiscoveryNode [id=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.31.70.152],
sockAddrs=[ip-172-31-70-152.ec2.internal/172.31.70.152:47500,
/0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=3,
intOrder=3, lastExchangeTime=1509377011758, loc=false,
ver=2.2.0#20170915-sha1:5747ce6b, isClient=false], connected=true,
connectCnt=0, queueLimit=4096, reserveCnt=1, pairedConnections=false],
super=GridNioSessionImpl [locAddr=/172.31.77.164:47100,
rmtAddr=/172.31.70.152:53298, createTime=1509377012153, closeTime=0,
bytesSent=41940044, bytesRcvd=10314443201, bytesSent0=219408,
bytesRcvd0=101826439, sndSchedTime=1509377012153, lastSndTime=1509377465319,
lastRcvTime=1509377465329, readsPaused=false,
filterChain=FilterChain[filters=[GridNioCodecFilter
[parser=o.a.i.i.util.nio.GridDirectParser@190e3b3d, directMode=true],
GridConnectionBytesVerifyFilter], accepted=true]]]
java.lang.OutOfMemoryError: Java heap space
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2$1.create(DirectByteBufferStreamImplV2.java:92)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2$1.create(DirectByteBufferStreamImplV2.java:82)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2.readArray(DirectByteBufferStreamImplV2.java:1467)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2.readByteArray(DirectByteBufferStreamImplV2.java:991)
at
org.apache.ignite.internal.direct.DirectMessageReader.readByteArray(DirectMessageReader.java:179)
at
org.apache.ignite.internal.GridJobExecuteRequest.readFrom(GridJobExecuteRequest.java:723)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2.readMessage(DirectByteBufferStreamImplV2.java:1165)
at
org.apache.ignite.internal.direct.DirectMessageReader.readMessage(DirectMessageReader.java:311)
at
org.apache.ignite.internal.managers.communication.GridIoMessage.readFrom(GridIoMessage.java:261)
at
org.apache.ignite.internal.util.nio.GridDirectParser.decode(GridDirectParser.java:90)
at
org.apache.ignite.internal.util.nio.GridNioCodecFilter.onMessageReceived(GridNioCodecFilter.java:114)
at
org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at
org.apache.ignite.internal.util.nio.GridConnectionBytesVerifyFilter.onMessageReceived(GridConnectionBytesVerifyFilter.java:133)
at
org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at
org.apache.ignite.internal.util.nio.GridNioServer$HeadFilter.onMessageReceived(GridNioServer.java:3222)
at
org.apache.ignite.internal.util.nio.GridNioFilterChain.onMessageReceived(GridNioFilterChain.java:175)
at
org.apache.ignite.internal.util.nio.GridNioServer$DirectNioClientWorker.processRead(GridNioServer.java:1213)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.processSelectedKeysOptimized(GridNioServer.java:2180)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:1968)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1669)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:748)
Oct 30, 2017 3:31:07 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Closing NIO session because of unhandled exception.
class org.apache.ignite.internal.util.nio.GridNioException: Java heap space
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.processSelectedKeysOptimized(GridNioServer.java:2199)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:1968)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1669)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2$1.create(DirectByteBufferStreamImplV2.java:92)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2$1.create(DirectByteBufferStreamImplV2.java:82)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2.readArray(DirectByteBufferStreamImplV2.java:1467)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2.readByteArray(DirectByteBufferStreamImplV2.java:991)
at
org.apache.ignite.internal.direct.DirectMessageReader.readByteArray(DirectMessageReader.java:179)
at
org.apache.ignite.internal.GridJobExecuteRequest.readFrom(GridJobExecuteRequest.java:723)
at
org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2.readMessage(DirectByteBufferStreamImplV2.java:1165)
at
org.apache.ignite.internal.direct.DirectMessageReader.readMessage(DirectMessageReader.java:311)
at
org.apache.ignite.internal.managers.communication.GridIoMessage.readFrom(GridIoMessage.java:261)
at
org.apache.ignite.internal.util.nio.GridDirectParser.decode(GridDirectParser.java:90)
at
org.apache.ignite.internal.util.nio.GridNioCodecFilter.onMessageReceived(GridNioCodecFilter.java:114)
at
org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at
org.apache.ignite.internal.util.nio.GridConnectionBytesVerifyFilter.onMessageReceived(GridConnectionBytesVerifyFilter.java:133)
at
org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at
org.apache.ignite.internal.util.nio.GridNioServer$HeadFilter.onMessageReceived(GridNioServer.java:3222)
at
org.apache.ignite.internal.util.nio.GridNioFilterChain.onMessageReceived(GridNioFilterChain.java:175)
at
org.apache.ignite.internal.util.nio.GridNioServer$DirectNioClientWorker.processRead(GridNioServer.java:1213)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.processSelectedKeysOptimized(GridNioServer.java:2180)
... 4 more
Also see:
Oct 30, 2017 3:31:15 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Failed to read message [msg=GridIoMessage [plc=0, topic=null,
topicOrd=-1, ordered=false, timeout=0, skipOnTimeout=false, msg=null],
buf=java.nio.DirectByteBuffer[pos=53 lim=32768 cap=32768],
reader=DirectMessageReader [state=DirectMessageState [pos=0,
stack=[StateItem [stream=DirectByteBufferStreamImplV2
[baseOff=140455966714608, arrOff=-1, tmpArrOff=0, tmpArrBytes=0,
msgTypeDone=true, msg=GridJobExecuteRequest [subjId=null, sesId=null,
jobId=null, startTaskTime=0, timeout=0, taskName=null, userVer=null,
taskClsName=null, ldrParticipants=null, cpSpi=null,
createTime=1509377474412,
clsLdrId=b2d61ed6f51-04a0268a-8450-4c1f-8f26-d1fa1c5893a0, depMode=SHARED,
dynamicSiblings=false, forceLocDep=true, sesFullSup=false, internal=false,
part=0, topVer=null, execName=null], mapIt=null, it=null, arrPos=-1,
keyDone=false, readSize=-1, readItems=0, prim=0, primShift=0, uuidState=0,
uuidMost=0, uuidLeast=0, uuidLocId=0, lastFinished=true], state=0],
StateItem [stream=DirectByteBufferStreamImplV2 [baseOff=140455966714608,
arrOff=-1, tmpArrOff=0, tmpArrBytes=0, msgTypeDone=false, msg=null,
mapIt=null, it=null, arrPos=-1, keyDone=false, readSize=-1, readItems=0,
prim=0, primShift=0, uuidState=0, uuidMost=0, uuidLeast=0, uuidLocId=0,
lastFinished=true], state=0], null, null, null, null, null, null, null,
null]], lastRead=true], ses=GridSelectorNioSessionImpl
[worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=2,
bytesRcvd=59647, bytesSent=28, bytesRcvd0=32768, bytesSent0=0, select=true,
super=GridWorker [name=grid-nio-worker-tcp-comm-2, igniteInstanceName=null,
finished=false, hashCode=2032347436, interrupted=false,
runner=grid-nio-worker-tcp-comm-2-#19%null%]]],
writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768],
readBuf=java.nio.DirectByteBuffer[pos=53 lim=32768 cap=32768],
inRecovery=GridNioRecoveryDescriptor [acked=2410, resendCnt=3, rcvCnt=3138,
sentCnt=2413, reserved=true, lastAck=3136, nodeLeft=false,
node=TcpDiscoveryNode [id=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.31.70.152],
sockAddrs=[ip-172-31-70-152.ec2.internal/172.31.70.152:47500,
/0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=3,
intOrder=3, lastExchangeTime=1509377011758, loc=false,
ver=2.2.0#20170915-sha1:5747ce6b, isClient=false], connected=true,
connectCnt=5, queueLimit=4096, reserveCnt=6, pairedConnections=false],
outRecovery=GridNioRecoveryDescriptor [acked=2410, resendCnt=3, rcvCnt=3138,
sentCnt=2413, reserved=true, lastAck=3136, nodeLeft=false,
node=TcpDiscoveryNode [id=04a0268a-8450-4c1f-8f26-d1fa1c5893a0,
addrs=[0:0:0:0:0:0:0:1%lo, 127.0.0.1, 172.31.70.152],
sockAddrs=[ip-172-31-70-152.ec2.internal/172.31.70.152:47500,
/0:0:0:0:0:0:0:1%lo:47500, /127.0.0.1:47500], discPort=47500, order=3,
intOrder=3, lastExchangeTime=1509377011758, loc=false,
ver=2.2.0#20170915-sha1:5747ce6b, isClient=false], connected=true,
connectCnt=5, queueLimit=4096, reserveCnt=6, pairedConnections=false],
super=GridNioSessionImpl [locAddr=/172.31.77.164:56060,
rmtAddr=ip-172-31-70-152.ec2.internal/172.31.70.152:47100,
createTime=1509377474412, closeTime=0, bytesSent=0, bytesRcvd=32768,
bytesSent0=0, bytesRcvd0=32768, sndSchedTime=1509377474412,
lastSndTime=1509377474412, lastRcvTime=1509377474412, readsPaused=false,
filterChain=FilterChain[filters=[GridNioCodecFilter
[parser=o.a.i.i.util.nio.GridDirectParser@190e3b3d, directMode=true],
GridConnectionBytesVerifyFilter], accepted=false]]]
java.lang.OutOfMemoryError: Java heap space
Oct 30, 2017 3:31:16 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Closing NIO session because of unhandled exception.
class org.apache.ignite.internal.util.nio.GridNioException: Java heap space
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.processSelectedKeysOptimized(GridNioServer.java:2199)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:1968)
at
org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1669)
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.OutOfMemoryError: Java heap space
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
