Hi,
I was running a performance test in which I tried to add a backlog of around
4-5 M messages over 10 virtual topic,pushed by around 100 producers and then
tried to drain them by equal number of consumers , with 10 consumers forming
a group of virtual topic consumer. in middle of draining I encountered with
following exception in broker log. Can anyone please tell me what went
wrong?? After the exception as well, the broker and consumers continued
their normal operations.
2011-08-11 02:36:19,067 ERROR
[org.apache.activemq.broker.region.cursors.AbstractStoreCursor]
QueueStorePrefetch1976196437 - Failed to fill batch
java.lang.NullPointerException
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110)
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121)
at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485)
at
org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88)
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713)
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932)
at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440)
at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at
org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
2011-08-11 02:36:19,070 ERROR
[org.apache.activemq.broker.region.cursors.AbstractStoreCursor]
QueueStorePrefetch1976196437 - Failed to fill batch
java.lang.RuntimeException: java.lang.NullPointerException
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713)
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932)
at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440)
at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at
org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.NullPointerException
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110)
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121)
at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485)
at
org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88)
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260)
... 25 more
2011-08-11 02:36:19,071 ERROR [org.apache.activemq.broker.region.Queue]
Failed to page in more queue messages
java.lang.RuntimeException: java.lang.RuntimeException:
java.lang.NullPointerException
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:148)
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713)
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932)
at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440)
at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at
org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
... 24 more
Caused by: java.lang.NullPointerException
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110)
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121)
at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485)
at
org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88)
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260)
... 25 more
2011-08-11 02:36:19,087 ERROR
[org.apache.activemq.broker.region.cursors.AbstractStoreCursor]
QueueStorePrefetch1976196437 - Failed to fill batch
java.lang.NullPointerException
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110)
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121)
at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485)
at
org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88)
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713)
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932)
at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440)
at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at
org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
2011-08-11 02:36:19,089 ERROR
[org.apache.activemq.broker.region.cursors.AbstractStoreCursor]
QueueStorePrefetch1976196437 - Failed to fill batch
java.lang.RuntimeException: java.lang.NullPointerException
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713)
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932)
at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440)
at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at
org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.NullPointerException
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110)
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121)
at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485)
at
org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88)
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260)
... 25 more
2011-08-11 02:36:19,090 ERROR [org.apache.activemq.broker.region.Queue]
Failed to page in more queue messages
java.lang.RuntimeException: java.lang.RuntimeException:
java.lang.NullPointerException
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:148)
at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1713)
at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:1932)
at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1440)
at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1643)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:394)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:549)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:77)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:465)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:229)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at
org.apache.activemq.transport.ResponseCorrelator.onCommand(ResponseCorrelator.java:116)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.vm.VMTransport.iterate(VMTransport.java:218)
at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:127)
at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.RuntimeException: java.lang.NullPointerException
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:263)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
... 24 more
Caused by: java.lang.NullPointerException
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.findNextPage(BTreeNode.java:110)
at
org.apache.kahadb.index.BTreeNode$BTreeIterator.hasNext(BTreeNode.java:121)
at
org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.hasNext(MessageDatabase.java:2359)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$5.execute(KahaDBStore.java:491)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:485)
at
org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:88)
at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:97)
at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:260)
... 25 more
--Kaustubh
