kal patel created AMQ-5026: ------------------------------ Summary: Replicated LevelDB Store getting EOF exception Key: AMQ-5026 URL: https://issues.apache.org/jira/browse/AMQ-5026 Project: ActiveMQ Issue Type: Bug Components: activemq-leveldb-store Affects Versions: 5.10.0 Reporter: kal patel
3 brokers setup for ReplicatedLevelDB master/slave conf. running producer and slow consumer and doing failover by stopping master broker. After few failovers, the leveldb throws EOF exception while reading record from the store. using the latest source code from activemq 5.10 branch. The file that has the issue: [root@vmx0568 data]# ls -l /solidstate/ldbkp/000000001f4022e4.log -rw-r--r--. 1 root root 91500224 Jan 17 14:44 /solidstate/ldbkp/000000001f4022e4.log [root@vmx0568 data]# Client | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b41aac, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | Seq read: 260217 | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b423ee, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | Seq read: 260218 | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b42d02, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | Seq read: 260219 | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b43628, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | Seq read: 260220 | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b43f6a, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | Seq read: 260221 | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b4487e, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | Seq read: 260222 | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,118 | INFO | locator read: DataLocator(24b451a4, 2283) | org.apache.activemq.leveldb.LevelDBClient | main 2014-01-17 14:44:19,124 | INFO | No IOExceptionHandler registered, ignoring IO exception | org.apache.activemq.broker.BrokerService | LevelDB IOException handler. java.io.EOFException: File '/solidstate/ldbkp/000000001f4022e4.log' offset: 91500234 at org.apache.activemq.leveldb.RecordLog$LogReader.read(RecordLog.scala:299) at org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:564) at org.apache.activemq.leveldb.RecordLog$$anonfun$read$2.apply(RecordLog.scala:564) at org.apache.activemq.leveldb.RecordLog.get_reader(RecordLog.scala:554) at org.apache.activemq.leveldb.RecordLog.read(RecordLog.scala:564) at org.apache.activemq.leveldb.LevelDBClient.getMessage(LevelDBClient.scala:1318) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1257) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$queueCursor$1.apply(LevelDBClient.scala:1252) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1342) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1$$anonfun$apply$mcV$sp$12.apply(LevelDBClient.scala:1341) at org.apache.activemq.leveldb.LevelDBClient$RichDB.check$4(LevelDBClient.scala:323) at org.apache.activemq.leveldb.LevelDBClient$RichDB.cursorRange(LevelDBClient.scala:325) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1341) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1341) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1341) at org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:1019) at org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:1025) at org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:549) at org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:1025) at org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1340) at org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1252) at org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:713) at org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:758) at org.apache.activemq.broker.region.Queue.initialize(Queue.java:369) at org.apache.activemq.broker.region.DestinationFactoryImpl.createDestination(DestinationFactoryImpl.java:87) at org.apache.activemq.broker.region.AbstractRegion.createDestination(AbstractRegion.java:516) at org.apache.activemq.broker.jmx.ManagedQueueRegion.createDestination(ManagedQueueRegion.java:56) at org.apache.activemq.broker.region.AbstractRegion.addDestination(AbstractRegion.java:134) at org.apache.activemq.broker.region.RegionBroker.addDestination(RegionBroker.java:321) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.advisory.AdvisoryBroker.addDestination(AdvisoryBroker.java:169) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:167) at org.apache.activemq.broker.MutableBrokerFilter.addDestination(MutableBrokerFilter.java:172) at org.apache.activemq.broker.region.AbstractRegion.start(AbstractRegion.java:98) at org.apache.activemq.broker.region.RegionBroker.start(RegionBroker.java:190) at org.apache.activemq.broker.jmx.ManagedRegionBroker.start(ManagedRegionBroker.java:120) at org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182) at org.apache.activemq.broker.BrokerFilter.start(BrokerFilter.java:182) at org.apache.activemq.broker.TransactionBroker.start(TransactionBroker.java:120) at org.apache.activemq.broker.BrokerService$5.start(BrokerService.java:2150) at org.apache.activemq.broker.BrokerService.doStartBroker(BrokerService.java:648) at org.apache.activemq.broker.BrokerService.startBroker(BrokerService.java:632) at org.apache.activemq.broker.BrokerService.start(BrokerService.java:568) at org.apache.activemq.xbean.XBeanBrokerService.afterPropertiesSet(XBeanBrokerService.java:73) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeCustomInitMethod(AbstractAutowireCapableBeanFactory.java:1608) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1549) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1479) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:521) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:458) at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:295) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:223) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:292) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194) at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:628) at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:932) at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:479) at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64) at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52) at org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104) at org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104) at org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67) at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71) at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54) at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87) at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:57) at org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:150) at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:57) at org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.activemq.console.Main.runTaskClass(Main.java:262) at org.apache.activemq.console.Main.main(Main.java:115) 2014-01-17 14:44:19,156 | INFO | Stopped LevelDB[/solidstate/ldbkp] | org.apache.activemq.leveldb.LevelDBStore | LevelDB IOException handler. -- This message was sent by Atlassian JIRA (v6.1.5#6160)