[
https://issues.apache.org/jira/browse/CASSANDRA-1248?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12885316#action_12885316
]
David King commented on CASSANDRA-1248:
---------------------------------------
Not sure how much of this is helpful, but here are the sorrounding logs with
DEBUG logging:
{quote}
DEBUG [ROW-READ-STAGE:185] 2010-07-05 12:21:00,642 ReadVerbHandler.java (line
75) digest is d41d8cd98f00b204e9800998ecf8427e
DEBUG [ROW-READ-STAGE:185] 2010-07-05 12:21:00,672 ReadVerbHandler.java (line
93) Read key sr_pop_en-gb_no_over18; sending response to 14264920@/10.251.82.223
DEBUG [ROW-MUTATION-STAGE:8] 2010-07-05 12:21:00,642
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='reddit',
key='3qtik_c0tjtwf', modifications=[ColumnFamily(CommentVote
[date:false:1...@1278355826446131,ip:false:1...@1278355826446131,name:false:1...@1278355826446131,thing1_id:false:5...@1278355826446131,thing2_id:false:7...@1278355826446131,valid_thing:false:1...@1278355826446131,valid_user:false:1...@1278355826446131,])])
DEBUG [ROW-MUTATION-STAGE:13] 2010-07-05 12:21:00,670
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='04a594c18be3804bf2ad140a18292aaef76f8641',
modifications=[ColumnFamily(permacache
[76616c7565:false:4...@1278357660440485,])])
DEBUG [ROW-MUTATION-STAGE:13] 2010-07-05 12:21:00,673
RowMutationVerbHandler.java (line 73) RowMutation(table='permacache',
key='04a594c18be3804bf2ad140a18292aaef76f8641',
modifications=[ColumnFamily(permacache
[76616c7565:false:4...@1278357660440485,])]) applied. Sending response to
2890086@/10.250.138.47
DEBUG [ROW-MUTATION-STAGE:6] 2010-07-05 12:21:00,669
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='comments_sort_21188087__confidence',
modifications=[ColumnFamily(permacache
[76616c7565:false:1...@1278357660437987,])])
DEBUG [ROW-MUTATION-STAGE:6] 2010-07-05 12:21:00,673
RowMutationVerbHandler.java (line 73) RowMutation(table='permacache',
key='comments_sort_21188087__confidence',
modifications=[ColumnFamily(permacache
[76616c7565:false:1...@1278357660437987,])]) applied. Sending response to
14264934@/10.251.82.223
ERROR [ROW-READ-STAGE:180] 2010-07-05 12:21:00,669
DebuggableThreadPoolExecutor.java (line 101) Error in ThreadPoolExecutor
java.lang.ArrayIndexOutOfBoundsException
at
org.apache.cassandra.io.util.BufferedRandomAccessFile.read(BufferedRandomAccessFile.java:326)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:414)
at java.io.DataInputStream.readUTF(DataInputStream.java:609)
at java.io.RandomAccessFile.readUTF(RandomAccessFile.java:938)
at
org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.<init>(SSTableSliceIterator.java:125)
at
org.apache.cassandra.db.filter.SSTableSliceIterator.<init>(SSTableSliceIterator.java:59)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:63)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:859)
at
org.apache.cassandra.db.ColumnFamilyStore.cacheRow(ColumnFamilyStore.java:747)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:772)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
at org.apache.cassandra.db.Table.getRow(Table.java:381)
at
org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:56)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
DEBUG [ROW-MUTATION-STAGE:9] 2010-07-05 12:21:00,666
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='comments_sort_21188087__hot', modifications=[ColumnFamily(permacache
[76616c7565:false:1...@1278357660434824,])])
DEBUG [ROW-MUTATION-STAGE:18] 2010-07-05 12:21:00,665
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='comments_sort_21186905__controversy',
modifications=[ColumnFamily(permacache
[76616c7565:false:26...@1278357660427622,])])
DEBUG [ROW-MUTATION-STAGE:18] 2010-07-05 12:21:00,676
RowMutationVerbHandler.java (line 73) RowMutation(table='permacache',
key='comments_sort_21186905__controversy',
modifications=[ColumnFamily(permacache
[76616c7565:false:26...@1278357660427622,])]) applied. Sending response to
14264931@/10.251.82.223
DEBUG [pool-1-thread-205] 2010-07-05 12:21:00,665 CassandraServer.java (line
227) multiget_slice
DEBUG [ROW-MUTATION-STAGE:28] 2010-07-05 12:21:00,661
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='reddit',
key='2jpa_c0tk7r8', modifications=[ColumnFamily(CommentVote
[date:false:1...@1278357660430268,ip:false:1...@1278357660430268,name:false:2...@1278357660430268,thing1_id:false:4...@1278357660430268,thing2_id:false:7...@1278357660430268,valid_thing:false:1...@1278357660430268,valid_user:false:1...@1278357660430268,])])
DEBUG [ROW-READ-STAGE:184] 2010-07-05 12:21:00,660 ReadVerbHandler.java (line
75) digest is d41d8cd98f00b204e9800998ecf8427e
DEBUG [ROW-MUTATION-STAGE:11] 2010-07-05 12:21:00,660
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='4247f98369d200f3a53bea7cf24461eb860e1a0f',
modifications=[ColumnFamily(permacache
[76616c7565:false:2...@1278357660417184,])])
DEBUG [ROW-MUTATION-STAGE:11] 2010-07-05 12:21:00,677
RowMutationVerbHandler.java (line 73) RowMutation(table='permacache',
key='4247f98369d200f3a53bea7cf24461eb860e1a0f',
modifications=[ColumnFamily(permacache
[76616c7565:false:2...@1278357660417184,])]) applied. Sending response to
14264928@/10.251.82.223
ERROR [ROW-READ-STAGE:190] 2010-07-05 12:21:00,658
DebuggableThreadPoolExecutor.java (line 101) Error in ThreadPoolExecutor
java.lang.ArrayIndexOutOfBoundsException
at
org.apache.cassandra.io.util.BufferedRandomAccessFile.read(BufferedRandomAccessFile.java:326)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:414)
at java.io.DataInputStream.readUTF(DataInputStream.java:609)
at java.io.RandomAccessFile.readUTF(RandomAccessFile.java:938)
at
org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.<init>(SSTableSliceIterator.java:125)
at
org.apache.cassandra.db.filter.SSTableSliceIterator.<init>(SSTableSliceIterator.java:59)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:63)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:859)
at
org.apache.cassandra.db.ColumnFamilyStore.cacheRow(ColumnFamilyStore.java:747)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:772)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
at org.apache.cassandra.db.Table.getRow(Table.java:381)
at
org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:56)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
DEBUG [ROW-MUTATION-STAGE:12] 2010-07-05 12:21:00,655
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='last_comments_t3_cm600', modifications=[ColumnFamily(permacache
[76616c7565:false:1...@1278357660544809,])])
DEBUG [ROW-MUTATION-STAGE:12] 2010-07-05 12:21:00,678
RowMutationVerbHandler.java (line 73) RowMutation(table='permacache',
key='last_comments_t3_cm600', modifications=[ColumnFamily(permacache
[76616c7565:false:1...@1278357660544809,])]) applied. Sending response to
2890083@/10.250.138.47
ERROR [ROW-READ-STAGE:190] 2010-07-05 12:21:00,678 CassandraDaemon.java (line
83) Uncaught exception in thread Thread[ROW-READ-STAGE:190,5,main]
java.lang.ArrayIndexOutOfBoundsException
at
org.apache.cassandra.io.util.BufferedRandomAccessFile.read(BufferedRandomAccessFile.java:326)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:414)
at java.io.DataInputStream.readUTF(DataInputStream.java:609)
at java.io.RandomAccessFile.readUTF(RandomAccessFile.java:938)
at
org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.<init>(SSTableSliceIterator.java:125)
at
org.apache.cassandra.db.filter.SSTableSliceIterator.<init>(SSTableSliceIterator.java:59)
at
org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:63)
at
org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:859)
at
org.apache.cassandra.db.ColumnFamilyStore.cacheRow(ColumnFamilyStore.java:747)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:772)
at
org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
at org.apache.cassandra.db.Table.getRow(Table.java:381)
at
org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:56)
at
org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
DEBUG [ROW-MUTATION-STAGE:4] 2010-07-05 12:21:00,651
RowMutationVerbHandler.java (line 47) Applying RowMutation(table='permacache',
key='comments_sort_21188483__controversy',
modifications=[ColumnFamily(permacache
[76616c7565:false:10...@1278357660432250,])])
{quote}
> ArrayIndexOutOfBoundsException thrown when reading partially-written sstable
> ----------------------------------------------------------------------------
>
> Key: CASSANDRA-1248
> URL: https://issues.apache.org/jira/browse/CASSANDRA-1248
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Environment: Cassandra 0.6.3
> java version "1.6.0_0"
> IcedTea6 1.3.1 (6b12-0ubuntu6.6) Runtime Environment (build 1.6.0_0-b12)
> OpenJDK 64-Bit Server VM (build 1.6.0_0-b12, mixed mode)
> Ubuntu 8.10
> 6 node cluster
> RF==3
> Reporter: David King
> Fix For: 0.8
>
>
> A node died by running out of disk space. After growing the volume, trying to
> execute reads dies with:
> {quote}
> ERROR 11:02:51,957 Uncaught exception in thread
> Thread[ROW-READ-STAGE:10,5,main]
> java.lang.ArrayIndexOutOfBoundsException
> at
> org.apache.cassandra.io.util.BufferedRandomAccessFile.read(BufferedRandomAccessFile.java:326)
> at java.io.RandomAccessFile.readFully(RandomAccessFile.java:414)
> at java.io.DataInputStream.readUTF(DataInputStream.java:609)
> at java.io.RandomAccessFile.readUTF(RandomAccessFile.java:938)
> at
> org.apache.cassandra.db.filter.SSTableSliceIterator$ColumnGroupReader.<init>(SSTableSliceIterator.java:125)
> at
> org.apache.cassandra.db.filter.SSTableSliceIterator.<init>(SSTableSliceIterator.java:59)
> at
> org.apache.cassandra.db.filter.SliceQueryFilter.getSSTableColumnIterator(SliceQueryFilter.java:63)
> at
> org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:859)
> at
> org.apache.cassandra.db.ColumnFamilyStore.cacheRow(ColumnFamilyStore.java:747)
> at
> org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:772)
> at
> org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:739)
> at org.apache.cassandra.db.Table.getRow(Table.java:381)
> at
> org.apache.cassandra.db.SliceByNamesReadCommand.getRow(SliceByNamesReadCommand.java:56)
> at
> org.apache.cassandra.db.ReadVerbHandler.doVerb(ReadVerbHandler.java:70)
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:40)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> {quote}
> The clients then die with TimeoutExceptions.
> It might be better to return an exception to the client immediately instead
> of letting them die, and it would be helpful if this exception were
> accompanied with a message indicating *which* SStable is malformed.
> This command does *not* turn up the failed table (nor does the same with
> sstablekeys):
> {quote}
> r...@pmc08:/cassandra/data$ for x in $(ls -t */*-Data.db); do ls -lh $x;
> /usr/local/cassandra/bin/sstable2json $x > /dev/null; done
> {quote}
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
