[ https://issues.apache.org/jira/browse/CASSANDRA-6565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14050682#comment-14050682 ]
Dmitry Mitko edited comment on CASSANDRA-6565 at 7/2/14 8:40 PM: ----------------------------------------------------------------- The same issue with 2.0.7. We can't add more nodes into our cluster anymore. {noformat} WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:02:41,966 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:04,954 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,563 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) ERROR [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,565 StreamSession.java (line 420) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Streaming error occurred java.io.IOException: Too many retries for Header (cfId: 9d1ff8a6-e358-302c-8b7b-49ca458ae31b, #23, version: jb, estimated keys: 5888, transfer size: 26198229, compressed?: true) at org.apache.cassandra.streaming.StreamSession.doRetry(StreamSession.java:544) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:51) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) ... 4 more {noformat} and then {noformat} WARN [NonPeriodicTasks:1] 2014-07-02 21:46:14,451 StreamResultFuture.java (line 215) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Stream failed ERROR [main] 2014-07-02 21:46:14,452 CassandraDaemon.java (line 497) Exception encountered during startup java.lang.RuntimeException: Error during boostrap: Stream failed at org.apache.cassandra.dht.BootStrapper.bootstrap(BootStrapper.java:86) at org.apache.cassandra.service.StorageService.bootstrap(StorageService.java:1035) at org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:797) at org.apache.cassandra.service.StorageService.initServer(StorageService.java:615) at org.apache.cassandra.service.StorageService.initServer(StorageService.java:505) at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:362) at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:480) at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:569) Caused by: org.apache.cassandra.streaming.StreamException: Stream failed at org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) at com.google.common.util.concurrent.Futures$4.run(Futures.java:1160) at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:216) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:191) at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:331) at org.apache.cassandra.streaming.StreamSession.maybeCompleted(StreamSession.java:615) at org.apache.cassandra.streaming.StreamSession.taskCompleted(StreamSession.java:566) at org.apache.cassandra.streaming.StreamReceiveTask$OnCompletionRunnable.run(StreamReceiveTask.java:120) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) {noformat} and the end is {noformat} ERROR [Thread-31] 2014-07-02 21:46:16,479 CassandraDaemon.java (line 198) Exception in thread Thread[Thread-31,5,main] java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut down at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:61) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372) at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor.execute(DebuggableThreadPoolExecutor.java:136) at org.apache.cassandra.net.MessagingService.receive(MessagingService.java:709) at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:161) at org.apache.cassandra.net.IncomingTcpConnection.handleModernVersion(IncomingTcpConnection.java:130) at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:74) {noformat} Please, investigate this issue or help with some workaround to force the joining process somehow. Maybe, just ignore failed streams will be good idea? We are using replication factor = 3. was (Author: dv1983): The same issue. We can't add more nodes into our cluster anymore. {noformat} WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:02:41,966 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:04,954 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,563 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) ERROR [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,565 StreamSession.java (line 420) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Streaming error occurred java.io.IOException: Too many retries for Header (cfId: 9d1ff8a6-e358-302c-8b7b-49ca458ae31b, #23, version: jb, estimated keys: 5888, transfer size: 26198229, compressed?: true) at org.apache.cassandra.streaming.StreamSession.doRetry(StreamSession.java:544) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:51) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37) at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.AssertionError at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94) at java.io.InputStream.read(InputStream.java:170) at java.io.InputStream.skip(InputStream.java:222) at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117) at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89) at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47) ... 4 more {noformat} and then {noformat} WARN [NonPeriodicTasks:1] 2014-07-02 21:46:14,451 StreamResultFuture.java (line 215) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Stream failed ERROR [main] 2014-07-02 21:46:14,452 CassandraDaemon.java (line 497) Exception encountered during startup java.lang.RuntimeException: Error during boostrap: Stream failed at org.apache.cassandra.dht.BootStrapper.bootstrap(BootStrapper.java:86) at org.apache.cassandra.service.StorageService.bootstrap(StorageService.java:1035) at org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:797) at org.apache.cassandra.service.StorageService.initServer(StorageService.java:615) at org.apache.cassandra.service.StorageService.initServer(StorageService.java:505) at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:362) at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:480) at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:569) Caused by: org.apache.cassandra.streaming.StreamException: Stream failed at org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) at com.google.common.util.concurrent.Futures$4.run(Futures.java:1160) at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:216) at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:191) at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:331) at org.apache.cassandra.streaming.StreamSession.maybeCompleted(StreamSession.java:615) at org.apache.cassandra.streaming.StreamSession.taskCompleted(StreamSession.java:566) at org.apache.cassandra.streaming.StreamReceiveTask$OnCompletionRunnable.run(StreamReceiveTask.java:120) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) {noformat} and the end is {noformat} ERROR [Thread-31] 2014-07-02 21:46:16,479 CassandraDaemon.java (line 198) Exception in thread Thread[Thread-31,5,main] java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut down at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:61) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372) at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor.execute(DebuggableThreadPoolExecutor.java:136) at org.apache.cassandra.net.MessagingService.receive(MessagingService.java:709) at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:161) at org.apache.cassandra.net.IncomingTcpConnection.handleModernVersion(IncomingTcpConnection.java:130) at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:74) {noformat} Please, investigate this issue or help with some workaround to force the joining process somehow. Maybe, just ignore failed streams will be good idea? We are using replication factor = 3. > New node refuses to join the ring. > ---------------------------------- > > Key: CASSANDRA-6565 > URL: https://issues.apache.org/jira/browse/CASSANDRA-6565 > Project: Cassandra > Issue Type: Bug > Reporter: Shao-Chuan Wang > > We have 30 nodes in one DC, 25 nodes in another. We are running 2.0.1. > Two nodes are joining the ring, but one of them failed > ARN [STREAM-IN-/10.4.197.53] 2014-01-09 19:41:40,418 StreamResultFuture.java > (line 209) [Stream #e515d6e0-795d-11e3-b74a-b72892248056] Stream failed > ERROR [main] 2014-01-09 19:41:40,418 CassandraDaemon.java (line 459) > Exception encountered during startup > java.lang.RuntimeException: Error during boostrap: Stream failed > at > org.apache.cassandra.dht.BootStrapper.bootstrap(BootStrapper.java:86) > at > org.apache.cassandra.service.StorageService.bootstrap(StorageService.java:901) > at > org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:670) > at > org.apache.cassandra.service.StorageService.initServer(StorageService.java:529) > at > org.apache.cassandra.service.StorageService.initServer(StorageService.java:428) > at > org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:343) > at > org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:442) > at > org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:485) > Caused by: org.apache.cassandra.streaming.StreamException: Stream failed > at > org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:210) > at > org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:185) > at > org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:321) > at > org.apache.cassandra.streaming.StreamSession.complete(StreamSession.java:501) > at org.apache.cassandra.streaming.StreamSession.messageReceived(Stre > amSession.java:376) at > org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:293) > at java.lang.Thread.run(Thread.java:744) > ERROR [StorageServiceShutdownHook] 2014-01-09 19:41:40,428 > CassandraDaemon.java (line 185) Exception in thread > Thread[StorageServiceShutdownHook,5,main] > java.lang.NullPointerException > at > org.apache.cassandra.service.StorageService.stopRPCServer(StorageService.java:312) > at > org.apache.cassandra.service.StorageService.shutdownClientServers(StorageService.java:361) > at > org.apache.cassandra.service.StorageService.access$000(StorageService.java:96) > at > org.apache.cassandra.service.StorageService$1.runMayThrow(StorageService.java:494) > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > at java.lang.Thread.run(Thread.java:744) -- This message was sent by Atlassian JIRA (v6.2#6252)