[ https://issues.apache.org/jira/browse/CASSANDRA-11678?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15531121#comment-15531121 ]
Sharvanath Pathak commented on CASSANDRA-11678: ----------------------------------------------- We have reproduced the same case, and found out that the hint file truncated because there was a hard reboot around the time of last write to the file. I think we basically need to handle partially written hint files. Also, the CRC file does not exist in this case (probably because it crashed while writing the hints file). May be ignoring the hints file if the CRC file is absent can be a way to fix this? > cassandra crush when enable hints_compression > --------------------------------------------- > > Key: CASSANDRA-11678 > URL: https://issues.apache.org/jira/browse/CASSANDRA-11678 > Project: Cassandra > Issue Type: Bug > Components: Core, Local Write-Read Paths > Environment: Centos 7 > Reporter: Weijian Lin > Assignee: Blake Eggleston > Priority: Critical > > When I enable hints_compression and set the compression class to > LZ4Compressor,the > cassandra (v3.05, V3.5.0) will crush。That is a bug, or any conf is wrong? > *Exception in V 3.5.0 * > {code} > ERROR [HintsDispatcher:2] 2016-04-26 15:02:56,970 > HintsDispatchExecutor.java:225 - Failed to dispatch hints file > abc4dda2-b551-427e-bb0b-e383d4a392e1-1461654138963-1.hints: file is > corrupted ({}) > org.apache.cassandra.io.FSReadError: java.io.EOFException > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:284) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:254) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatcher.sendHints(HintsDispatcher.java:156) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatcher.sendHintsAndAwait(HintsDispatcher.java:137) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatcher.dispatch(HintsDispatcher.java:119) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatcher.dispatch(HintsDispatcher.java:91) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.deliver(HintsDispatchExecutor.java:259) > [apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.dispatch(HintsDispatchExecutor.java:242) > [apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.dispatch(HintsDispatchExecutor.java:220) > [apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.run(HintsDispatchExecutor.java:199) > [apache-cassandra-3.5.0.jar:3.5.0] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > [na:1.8.0_65] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_65] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [na:1.8.0_65] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [na:1.8.0_65] > at java.lang.Thread.run(Thread.java:745) [na:1.8.0_65] > Caused by: java.io.EOFException: null > at > org.apache.cassandra.io.util.RebufferingInputStream.readByte(RebufferingInputStream.java:146) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.io.util.RebufferingInputStream.readPrimitiveSlowly(RebufferingInputStream.java:108) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.io.util.RebufferingInputStream.readInt(RebufferingInputStream.java:188) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNextInternal(HintsReader.java:297) > ~[apache-cassandra-3.5.0.jar:3.5.0] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:280) > ~[apache-cassandra-3.5.0.jar:3.5.0] > ... 15 common frames omitted > {code} > *Exception in V 3.0.5 * > {code} > ERROR [HintsDispatcher:2] 2016-04-26 15:54:46,294 > HintsDispatchExecutor.java:225 - Failed to dispatch hints file > 8603be13-6878-4de3-8bc3-a7a7146b0376-1461657251205-1.hints: file is > corrupted ({}) > org.apache.cassandra.io.FSReadError: java.io.EOFException > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:282) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:252) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.sendHints(HintsDispatcher.java:156) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.sendHintsAndAwait(HintsDispatcher.java:137) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.dispatch(HintsDispatcher.java:119) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.dispatch(HintsDispatcher.java:91) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.deliver(HintsDispatchExecutor.java:259) > [apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.dispatch(HintsDispatchExecutor.java:242) > [apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.dispatch(HintsDispatchExecutor.java:220) > [apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.run(HintsDispatchExecutor.java:199) > [apache-cassandra-3.0.5.jar:3.0.5] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > [na:1.8.0_65] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_65] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [na:1.8.0_65] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [na:1.8.0_65] > at java.lang.Thread.run(Thread.java:745) [na:1.8.0_65] > Caused by: java.io.EOFException: null > at > org.apache.cassandra.io.util.RebufferingInputStream.readByte(RebufferingInputStream.java:146) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.io.util.RebufferingInputStream.readPrimitiveSlowly(RebufferingInputStream.java:108) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.io.util.RebufferingInputStream.readInt(RebufferingInputStream.java:188) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNextInternal(HintsReader.java:295) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:278) > ~[apache-cassandra-3.0.5.jar:3.0.5] > ... 15 common frames omitted > ERROR [HintsDispatcher:2] 2016-04-26 15:54:46,296 CassandraDaemon.java:195 > Exception in thread Thread[HintsDispatcher:2,1,main] > org.apache.cassandra.io.FSReadError: java.io.EOFException > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:282) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:252) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.sendHints(HintsDispatcher.java:156) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.sendHintsAndAwait(HintsDispatcher.java:137) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.dispatch(HintsDispatcher.java:119) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatcher.dispatch(HintsDispatcher.java:91) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.deliver(HintsDispatchExecutor.java:259) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.dispatch(HintsDispatchExecutor.java:242) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.dispatch(HintsDispatchExecutor.java:220) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsDispatchExecutor$DispatchHintsTask.run(HintsDispatchExecutor.java:199) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[na:1.8.0_65] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_65] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[na:1.8.0_65] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [na:1.8.0_65] > at java.lang.Thread.run(Thread.java:745) [na:1.8.0_65] > Caused by: java.io.EOFException: null > at > org.apache.cassandra.io.util.RebufferingInputStream.readByte(RebufferingInputStream.java:146) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.io.util.RebufferingInputStream.readPrimitiveSlowly(RebufferingInputStream.java:108) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.io.util.RebufferingInputStream.readInt(RebufferingInputStream.java:188) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNextInternal(HintsReader.java:295) > ~[apache-cassandra-3.0.5.jar:3.0.5] > at > org.apache.cassandra.hints.HintsReader$BuffersIterator.computeNext(HintsReader.java:278) > ~[apache-cassandra-3.0.5.jar:3.0.5] > ... 15 common frames omitted > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332)