[
https://issues.apache.org/jira/browse/CASSANDRA-2240?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13000060#comment-13000060
]
Sébastien Giroux commented on CASSANDRA-2240:
---------------------------------------------
I tried to activate debug log but since this is a live node, it seems there is
WAY too much going on. Here is the INFO log:
ERROR 23:10:35,105 Fatal exception in thread Thread[CompactionExecutor:1,1,main]
java.io.IOError:
org.apache.cassandra.db.ColumnSerializer$CorruptColumnException: invalid column
name length 0
at
org.apache.cassandra.io.util.ColumnIterator.deserializeNext(ColumnSortedMap.java:246)
at
org.apache.cassandra.io.util.ColumnIterator.next(ColumnSortedMap.java:262)
at
org.apache.cassandra.io.util.ColumnIterator.next(ColumnSortedMap.java:223)
at
java.util.concurrent.ConcurrentSkipListMap.buildFromSorted(ConcurrentSkipListMap.java:1521)
at
java.util.concurrent.ConcurrentSkipListMap.<init>(ConcurrentSkipListMap.java:1471)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:366)
at
org.apache.cassandra.db.SuperColumnSerializer.deserialize(SuperColumn.java:314)
at
org.apache.cassandra.db.ColumnFamilySerializer.deserializeColumns(ColumnFamilySerializer.java:129)
at
org.apache.cassandra.io.sstable.SSTableIdentityIterator.getColumnFamilyWithColumns(SSTableIdentityIterator.java:172)
at
org.apache.cassandra.io.PrecompactedRow.<init>(PrecompactedRow.java:78)
at
org.apache.cassandra.io.CompactionIterator.getCompactedRow(CompactionIterator.java:139)
at
org.apache.cassandra.io.CompactionIterator.getReduced(CompactionIterator.java:108)
at
org.apache.cassandra.io.CompactionIterator.getReduced(CompactionIterator.java:43)
at
org.apache.cassandra.utils.ReducingIterator.computeNext(ReducingIterator.java:73)
at
com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:136)
at
com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:131)
at
org.apache.commons.collections.iterators.FilterIterator.setNextObject(FilterIterator.java:183)
at
org.apache.commons.collections.iterators.FilterIterator.hasNext(FilterIterator.java:94)
at
org.apache.cassandra.db.CompactionManager.doCompaction(CompactionManager.java:448)
at
org.apache.cassandra.db.CompactionManager$1.call(CompactionManager.java:123)
at
org.apache.cassandra.db.CompactionManager$1.call(CompactionManager.java:93)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: org.apache.cassandra.db.ColumnSerializer$CorruptColumnException:
invalid column name length 0
at
org.apache.cassandra.db.ColumnSerializer.deserialize(ColumnSerializer.java:68)
at
org.apache.cassandra.io.util.ColumnIterator.deserializeNext(ColumnSortedMap.java:242)
... 25 more
INFO 23:10:35,106 Scrubbing
SSTableReader(path='/var/lib/cassandra/data/Wajam/Comment-f-710-Data.db')
ERROR 23:10:37,489 Fatal exception in thread Thread[CompactionExecutor:1,1,main]
java.io.EOFException
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:416)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:394)
at
org.apache.cassandra.io.util.BufferedRandomAccessFile.readBytes(BufferedRandomAccessFile.java:268)
at
org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:310)
at
org.apache.cassandra.utils.ByteBufferUtil.readWithShortLength(ByteBufferUtil.java:284)
at
org.apache.cassandra.db.CompactionManager.doScrub(CompactionManager.java:540)
at
org.apache.cassandra.db.CompactionManager.access$600(CompactionManager.java:55)
at
org.apache.cassandra.db.CompactionManager$3.call(CompactionManager.java:194)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
And the exception that is output by nodetool scrub:
Error occured while scrubbing keyspace <keyspacename>
java.util.concurrent.ExecutionException: java.io.EOFException
at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:252)
at java.util.concurrent.FutureTask.get(FutureTask.java:111)
at
org.apache.cassandra.db.CompactionManager.performScrub(CompactionManager.java:203)
at
org.apache.cassandra.db.ColumnFamilyStore.scrub(ColumnFamilyStore.java:963)
at
org.apache.cassandra.service.StorageService.scrub(StorageService.java:1247)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:111)
at
com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:45)
at
com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:226)
at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:251)
at
com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:857)
at
com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:795)
at
javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1450)
at
javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:90)
at
javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1285)
at
javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1383)
at
javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:807)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322)
at sun.rmi.transport.Transport$1.run(Transport.java:177)
at java.security.AccessController.doPrivileged(Native Method)
at sun.rmi.transport.Transport.serviceCall(Transport.java:173)
at
sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:553)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:808)
at
sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:667)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Caused by: java.io.EOFException
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:416)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:394)
at
org.apache.cassandra.io.util.BufferedRandomAccessFile.readBytes(BufferedRandomAccessFile.java:268)
at
org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:310)
at
org.apache.cassandra.utils.ByteBufferUtil.readWithShortLength(ByteBufferUtil.java:284)
at
org.apache.cassandra.db.CompactionManager.doScrub(CompactionManager.java:540)
at
org.apache.cassandra.db.CompactionManager.access$600(CompactionManager.java:55)
at
org.apache.cassandra.db.CompactionManager$3.call(CompactionManager.java:194)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
... 3 more
> nodetool scrub hangs or throws an exception
> -------------------------------------------
>
> Key: CASSANDRA-2240
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2240
> Project: Cassandra
> Issue Type: Bug
> Components: Tools
> Affects Versions: 0.7.3
> Environment: using build #314 from hudson
> Reporter: Yaniv Kunda
> Assignee: Jonathan Ellis
> Fix For: 0.7.3
>
> Attachments: exception2.txt, jstack1.txt,
> signatureBuckets-f-104.tar.gz, test-0.6.x-tables.tar.gz,
> userChannelFilter-f-210.tar.gz
>
>
> trying to run nodetool scrub hung or (only happened one time) threw the
> following exception:
> Error occured while scrubbing keyspace <mykeyspace>
> java.util.concurrent.ExecutionException: java.lang.AssertionError
> at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:252)
> at java.util.concurrent.FutureTask.get(FutureTask.java:111)
> at
> org.apache.cassandra.db.CompactionManager.performScrub(CompactionManager.java:203)
> at org.apache.cassandra.db.ColumnFamilyStore.scrub(ColumnFamilyStore.java:934)
> at org.apache.cassandra.service.StorageService.scrub(StorageService.java:1247)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:616)
> at
> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:111)
> at
> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:45)
> at
> com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:226)
> at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
> at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:251)
> at
> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:857)
> at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:795)
> at
> javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1450)
> at
> javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:90)
> at
> javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1285)
> at
> javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1383)
> at
> javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:807)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:616)
> at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:322)
> at sun.rmi.transport.Transport$1.run(Transport.java:177)
> at java.security.AccessController.doPrivileged(Native Method)
> at sun.rmi.transport.Transport.serviceCall(Transport.java:173)
> at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:553)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:808)
> at
> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:667)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> Caused by: java.lang.AssertionError
> at
> org.apache.cassandra.dht.RandomPartitioner.convertFromDiskFormat(RandomPartitioner.java:62)
> at
> org.apache.cassandra.io.sstable.SSTableReader.decodeKey(SSTableReader.java:627)
> at
> org.apache.cassandra.db.CompactionManager.doScrub(CompactionManager.java:541)
> at
> org.apache.cassandra.db.CompactionManager.access$600(CompactionManager.java:55)
> at
> org.apache.cassandra.db.CompactionManager$3.call(CompactionManager.java:194)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> ... 3 more
--
This message is automatically generated by JIRA.
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
