[ https://issues.apache.org/jira/browse/CASSANDRA-6129?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13786008#comment-13786008 ]
koray sariteke edited comment on CASSANDRA-6129 at 10/4/13 9:05 AM: -------------------------------------------------------------------- At BulkLoad process, get exception from StreamFuture eventListeners list as: INFO 17:27:46,987 [Stream #f92ac500-2c37-11e3-ad9c-99a3e9e32246] Prepare completed. Receiving 0 files(0 bytes), sending 1 files(115 bytes) ERROR 17:27:47,005 Error in ThreadPoolExecutor java.util.ConcurrentModificationException at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.next(ArrayList.java:791) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:781) ERROR 17:27:47,005 Error in ThreadPoolExecutor java.util.ConcurrentModificationException at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.next(ArrayList.java:791) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:781) Exception in thread "StreamConnectionEstablisher:1" Exception in thread "StreamConnectionEstablisher:2" java.util.ConcurrentModificationExceptionjava.util.ConcurrentModificationException at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.next(ArrayList.java:791) at java.util.ArrayList$Itr.next(ArrayList.java:791) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:781) at java.lang.Thread.run(Thread.java:781) Terminated was (Author: ksaritek): Get exception from StreamFuture eventListeners list as: INFO 17:27:46,987 [Stream #f92ac500-2c37-11e3-ad9c-99a3e9e32246] Prepare completed. Receiving 0 files(0 bytes), sending 1 files(115 bytes) ERROR 17:27:47,005 Error in ThreadPoolExecutor java.util.ConcurrentModificationException at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.next(ArrayList.java:791) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:781) ERROR 17:27:47,005 Error in ThreadPoolExecutor java.util.ConcurrentModificationException at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.next(ArrayList.java:791) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:781) Exception in thread "StreamConnectionEstablisher:1" Exception in thread "StreamConnectionEstablisher:2" java.util.ConcurrentModificationExceptionjava.util.ConcurrentModificationException at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:819) at java.util.ArrayList$Itr.next(ArrayList.java:791) at java.util.ArrayList$Itr.next(ArrayList.java:791) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:197) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionPrepared(StreamResultFuture.java:175) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.startStreamingFiles(StreamSession.java:620) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession.onInitializationComplete(StreamSession.java:400) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at org.apache.cassandra.streaming.StreamSession$1.run(StreamSession.java:200) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:781) at java.lang.Thread.run(Thread.java:781) Terminated > get java.util.ConcurrentModificationException while bulkloading from sstable > for widerow table > ---------------------------------------------------------------------------------------------- > > Key: CASSANDRA-6129 > URL: https://issues.apache.org/jira/browse/CASSANDRA-6129 > Project: Cassandra > Issue Type: Bug > Components: Core, Tools > Environment: three cassandra 2.0.1 node > jdk 7 > linux - ubuntu > Reporter: koray sariteke > Assignee: Jonathan Ellis > Fix For: 2.0.2 > > Attachments: BulkLoader.diff > > > I haven't faced that problem with cassandra 1.2.6 > I have created widerow sstables with SSTableSimpleUnsortedWriter. When i > tried to load sstables by sstableloader, I got > java.util.ConcurrentModificationException after a while (not at the beggining > of the streaming). > Exception is : > progress: [/192.168.103.5 0/39 (0%)] [/192.168.103.3 0/39 (0%)] > [/192.168.103.1 0/39 (0%)] [total: 0% - 15MB/s (avg: 0MB/s)] INFO > 00:45:23,542 [Stream #c0f53e00-2ae2-11e3-ab6b-99a3e9e32246] Session with > /192.168.103.3 is complete > progress: [/192.168.103.5 0/39 (0%)] [/192.168.103.3 0/39 (0%)] > [/192.168.103.1 0/39 (0%)] [total: 0% - 3MB/s (avg: 1MB/s)]Exception in > thread "STREAM-OUT-/192.168.103.3" java.util.ConcurrentModificationException > at java.util.HashMap$HashIterator.nextEntry(HashMap.java:894) > at java.util.HashMap$EntryIterator.next(HashMap.java:934) > at java.util.HashMap$EntryIterator.next(HashMap.java:932) > at > org.apache.cassandra.tools.BulkLoader$ProgressIndicator.handleStreamEvent(BulkLoader.java:129) > at > org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:198) > at > org.apache.cassandra.streaming.StreamResultFuture.handleProgress(StreamResultFuture.java:191) > at > org.apache.cassandra.streaming.StreamSession.progress(StreamSession.java:474) > at > org.apache.cassandra.streaming.StreamWriter.write(StreamWriter.java:105) > at > org.apache.cassandra.streaming.messages.FileMessage$1.serialize(FileMessage.java:73) > at > org.apache.cassandra.streaming.messages.FileMessage$1.serialize(FileMessage.java:45) > at > org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) > at > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:384) > at > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:357) > at java.lang.Thread.run(Thread.java:781) > progress: [/192.168.103.5 0/39 (3%)] [/192.168.103.3 0/39 (0%)] > [/192.168.103.1 0/39 (2%)] [total: 1% - 2147483647MB/s (avg: > 12MB/s)]Exception in thread "STREAM-OUT-/192.168.103.1" > java.util.ConcurrentModificationException > at java.util.HashMap$HashIterator.nextEntry(HashMap.java:894) > at java.util.HashMap$KeyIterator.next(HashMap.java:928) > progress: [/192.168.103.5 0/39 (3%)] [/192.168.103.3 0/39 (0%)] > [/192.168.103.1 0/39 (2%)] [total: 1% - 2147483647MB/s (avg: 12MB/s)] > at > org.apache.cassandra.streaming.StreamResultFuture.fireStreamEvent(StreamResultFuture.java:198) > at > org.apache.cassandra.streaming.StreamResultFuture.handleProgress(StreamResultFuture.java:191) > at > org.apache.cassandra.streaming.StreamSession.progress(StreamSession.java:474) > at > org.apache.cassandra.streaming.StreamWriter.write(StreamWriter.java:105) > at > org.apache.cassandra.streaming.messages.FileMessage$1.serialize(FileMessage.java:73) > at > org.apache.cassandra.streaming.messages.FileMessage$1.serialize(FileMessage.java:45) > at > org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) > at > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:384) > at > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:357) > at java.lang.Thread.run(Thread.java:781) -- This message was sent by Atlassian JIRA (v6.1#6144)