[ https://issues.apache.org/jira/browse/CASSANDRA-10689?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15000901#comment-15000901 ]
mlowicki commented on CASSANDRA-10689: -------------------------------------- After upgrade from 2.1.9 to 2.1.11 two days ago I'm getting lots of: {code} WARN [SharedPool-Worker-28] 2015-11-11 19:01:22,409 AbstractTracingAwareExecutorService.java:169 - Uncaught exception on thread Thread[SharedPool-Worker-28,5,main]: {} org.apache.cassandra.io.sstable.CorruptSSTableException: org.apache.cassandra.io.compress.CorruptBlockException: (/var/lib/cassandra/data2/sync/entity2-e24b5040199b11e5a30f75bb514ae072/sync-entity2-ka-392603-Data.db): corruption detected, chunk at 11612338 of length 156219476. at org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:85) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:310) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.io.util.PoolingSegmentedFile.getSegment(PoolingSegmentedFile.java:64) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.io.sstable.SSTableReader.getFileDataInput(SSTableReader.java:1894) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.columniterator.IndexedSliceReader.setToRowStart(IndexedSliceReader.java:107) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:83) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:65) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:42) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:246) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:62) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:270) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:62) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1994) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1837) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.Keyspace.getRow(Keyspace.java:353) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:85) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:47) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64) ~[apache-cassandra-2.1.11.jar:2.1.11] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_80] at org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) [apache-cassandra-2.1.11.jar:2.1.11] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80] Caused by: org.apache.cassandra.io.compress.CorruptBlockException: (/var/lib/cassandra/data2/sync/entity2-e24b5040199b11e5a30f75bb514ae072/sync-entity2-ka-392603-Data.db): corruption detected, chunk at 11612338 of length 156219476. at org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:116) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:81) ~[apache-cassandra-2.1.11.jar:2.1.11] ... 21 common frames omitted Caused by: java.io.IOException: Compressed lengths mismatch at org.apache.cassandra.io.compress.LZ4Compressor.uncompress(LZ4Compressor.java:98) ~[apache-cassandra-2.1.11.jar:2.1.11] at org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:112) ~[apache-cassandra-2.1.11.jar:2.1.11] ... 22 common frames omitted {code} On 3 out of 7 nodes in one data center. > java.lang.OutOfMemoryError: Direct buffer memory > ------------------------------------------------ > > Key: CASSANDRA-10689 > URL: https://issues.apache.org/jira/browse/CASSANDRA-10689 > Project: Cassandra > Issue Type: Bug > Reporter: mlowicki > > {code} > ERROR [SharedPool-Worker-63] 2015-11-11 17:53:16,161 > JVMStabilityInspector.java:117 - JVM state determined to be unstable. > Exiting forcefully due to: > java.lang.OutOfMemoryError: Direct buffer memory > at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.7.0_80] > at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) > ~[na:1.7.0_80] > at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306) > ~[na:1.7.0_80] > at sun.nio.ch.Util.getTemporaryDirectBuffer(Util.java:174) > ~[na:1.7.0_80] > at sun.nio.ch.IOUtil.read(IOUtil.java:195) ~[na:1.7.0_80] > at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:149) > ~[na:1.7.0_80] > at > org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(CompressedRandomAccessReader.java:104) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer(CompressedRandomAccessReader.java:81) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.io.util.RandomAccessReader.seek(RandomAccessReader.java:310) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.io.util.PoolingSegmentedFile.getSegment(PoolingSegmentedFile.java:64) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.io.sstable.SSTableReader.getFileDataInput(SSTableReader.java:1894) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.columniterator.IndexedSliceReader.setToRowStart(IndexedSliceReader.java:107) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.columniterator.IndexedSliceReader.<init>(IndexedSliceReader.java:83) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.columniterator.SSTableSliceIterator.createReader(SSTableSliceIterator.java:65) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.columniterator.SSTableSliceIterator.<init>(SSTableSliceIterator.java:42) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:246) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.filter.QueryFilter.getSSTableColumnIterator(QueryFilter.java:62) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:270) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:62) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1994) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1837) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at org.apache.cassandra.db.Keyspace.getRow(Keyspace.java:353) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:85) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:47) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > ~[na:1.7.0_80] > at > org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) > ~[apache-cassandra-2.1.11.jar:2.1.11] > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) > [apache-cassandra-2.1.11.jar:2.1.11] > at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80] > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332)