[ https://issues.apache.org/jira/browse/CASSANDRA-13087?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15949861#comment-15949861 ]
Paulo Motta commented on CASSANDRA-13087: ----------------------------------------- This is probably a leftover from the bug CASSANDRA-10791 and is be fixed by scrubbing the faulty sstables on the source and destination nodes before running repair again. > Not enough bytes exception during compaction > -------------------------------------------- > > Key: CASSANDRA-13087 > URL: https://issues.apache.org/jira/browse/CASSANDRA-13087 > Project: Cassandra > Issue Type: Bug > Components: Compaction > Environment: Ubuntu 14.04.3 LTS, Cassandra 2.1.14 > Reporter: FACORAT > Attachments: CASSANDRA-13087.patch > > > After a repair we have compaction exceptions on some nodes and its spreading > {noformat} > ERROR [CompactionExecutor:14065] 2016-12-30 14:45:07,245 > CassandraDaemon.java:229 - Exception in thread > Thread[CompactionExecutor:14065,1,main] > java.lang.IllegalArgumentException: Not enough bytes. Offset: 5. Length: > 20275. Buffer size: 12594 > at > org.apache.cassandra.db.composites.AbstractCType.checkRemaining(AbstractCType.java:378) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.composites.AbstractCompoundCellNameType.fromByteBuffer(AbstractCompoundCellNameType.java:100) > ~[apache-cassandra-2.1.14.ja > r:2.1.14] > at > org.apache.cassandra.db.composites.AbstractCType$Serializer.deserialize(AbstractCType.java:398) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.composites.AbstractCType$Serializer.deserialize(AbstractCType.java:382) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:75) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.AbstractCell$1.computeNext(AbstractCell.java:52) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.AbstractCell$1.computeNext(AbstractCell.java:46) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143) > ~[guava-16.0.jar:na] > at > com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138) > ~[guava-16.0.jar:na] > at > org.apache.cassandra.io.sstable.SSTableIdentityIterator.hasNext(SSTableIdentityIterator.java:171) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext(MergeIterator.java:202) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143) > ~[guava-16.0.jar:na] > at > com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138) > ~[guava-16.0.jar:na] > at > com.google.common.collect.Iterators$7.computeNext(Iterators.java:645) > ~[guava-16.0.jar:na] > at > com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143) > ~[guava-16.0.jar:na] > at > com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138) > ~[guava-16.0.jar:na] > at > org.apache.cassandra.db.ColumnIndex$Builder.buildForCompaction(ColumnIndex.java:166) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.LazilyCompactedRow.write(LazilyCompactedRow.java:121) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.io.sstable.SSTableWriter.append(SSTableWriter.java:193) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.io.sstable.SSTableRewriter.append(SSTableRewriter.java:127) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionTask.runMayThrow(CompactionTask.java:197) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionTask.executeInternal(CompactionTask.java:73) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.AbstractCompactionTask.execute(AbstractCompactionTask.java:59) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionManager$BackgroundCompactionCandidate.run(CompactionManager.java:264) > ~[apache-cassandra-2.1.14.jar:2 > .1.14] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[na:1.8.0_60] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > ~[na:1.8.0_60] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > ~[na:1.8.0_60] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [na:1.8.0_60] > at java.lang.Thread.run(Thread.java:745) [na:1.8.0_60] > {noformat} > nodetool scrub will discard part of the sstable with the following errors: > {noformat} > WARN [CompactionExecutor:14074] 2016-12-30 16:32:31,290 > OutputHandler.java:57 - Error reading row (stacktrace follows): > java.io.IOError: java.io.IOException: Key from data file () does not match > key from index file > (0014434c50303130303030303030363131373833313900000350494400) > at > org.apache.cassandra.db.compaction.Scrubber.scrub(Scrubber.java:213) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionManager.scrubOne(CompactionManager.java:721) > [apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionManager.access$300(CompactionManager.java:90) > [apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionManager$3.execute(CompactionManager.java:375) > [apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.compaction.CompactionManager$2.call(CompactionManager.java:320) > [apache-cassandra-2.1.14.jar:2.1.14] > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > [na:1.8.0_60] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [na:1.8.0_60] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [na:1.8.0_60] > at java.lang.Thread.run(Thread.java:745) [na:1.8.0_60] > Caused by: java.io.IOException: Key from data file () does not match key from > index file (0014434c50303130303030303030363131373833313900000350494400) > ... 9 common frames omitted > INFO [CompactionExecutor:14074] 2016-12-30 16:32:31,291 > OutputHandler.java:42 - Retrying from row index; data is -29 bytes starting > at 58994421 > WARN [CompactionExecutor:14074] 2016-12-30 16:32:31,291 > OutputHandler.java:57 - Retry failed too. Skipping to next row (retry's > stacktrace follows) > java.lang.IllegalArgumentException: Not enough bytes. Offset: 2. Length: > 21505. Buffer size: 17477 > at > org.apache.cassandra.db.composites.AbstractCType.checkRemaining(AbstractCType.java:378) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.composites.AbstractCompoundCellNameType.fromByteBuffer(AbstractCompoundCellNameType.java:100) > ~[apache-cassandra-2.1.14.ja > r:2.1.14] > at > org.apache.cassandra.db.composites.AbstractCType$Serializer.deserialize(AbstractCType.java:398) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.composites.AbstractCType$Serializer.deserialize(AbstractCType.java:382) > ~[apache-cassandra-2.1.14.jar:2.1.14] > at > org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:75) > ~[apache-cassandra-2.1.14.jar:2.1.14] > {noformat} -- This message was sent by Atlassian JIRA (v6.3.15#6346)