Re: Error while rebuilding a node: Stream failed
gt;>>>>>> StreamSession.java:505 - [Stream >>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>> >>>>>>>>>>> Streaming error occurred >>>> >>>>>>>>>>> java.io.IOException: Connection timed out >>>> >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native >>>> Method) >>>> >>>>>>>>>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>> >>>>>>>>>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown >>>> >>>>>>>>>>> Source) ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) >>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown >>>> Source) >>>> >>>>>>>>>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) >>>> [na:1.7.0_79] >>>> >>>>>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>> >>>>>>>>>>> StreamResultFuture.java:180 - [Stream >>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>> >>>>>>>>>>> Session with /192.168.1.140 is complete >>>> >>>>>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>> >>>>>>>>>>> StreamResultFuture.java:207 - [Stream >>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>> >>>>>>>>>>> Stream failed >>>> >>>>>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 >>>> 22:44:58,628 >>>> >>>>>>>>>>> StorageService.java:1075 - Error while rebuilding node >>>> >>>>>>>>>>> org.apache.cassandra.streaming.StreamException: Stream >>>> failed >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>>> >>>>>>>>>>> ~[guava-16.0.jar:na] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>>> >>>>>>>>>>> ~[guava-16.0.jar:na] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) >>>> >>>>>>>>>>> ~[guava-16.0.jar:na] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) >>>> >>>>>>>>>>> ~[guava-16.0.jar:na] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) >>>> >>>>>>>>>>> ~[guava-16.0.jar:na] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) >>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 >>>> >>>>>>>>>>> StreamSession.java:505 - [Stream >>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>> >>>>>>>>>>> Streaming error occurred >>>> >>>>>>>>>>> java.io.IOException: Broken pipe >>>> >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native >>>> Method) >>>> >>>>>>>>>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>> >>>>>>>>>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown >>>> >>>>>>>>>>> Source) ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) >>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown >>>> Source) >>>> >>>>>>>>>>> ~[na:1.7.0_79] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at >>>> >>>>>>>>>>> >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) >>>> [na:1.7.0_79] >>>> >>>> >>>> >>>> -- >>>> Eric Evans >>>> john.eric.ev...@gmail.com >>>> >>> >>> >> >
Re: Error while rebuilding a node: Stream failed
t >>> >>>> >>> org.apache.cassandra.streaming.StreamSession.receive(StreamSession.java:568) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:457) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:263) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>> >>>> >>> >>>> And this is from the source node: >>> >>>> >>> >>>> ERROR [STREAM-OUT-/172.31.22.104] 2016-05-26 11:08:05,097 >>> >>>> StreamSession.java:505 - [Stream >>> #74c57bc0-231a-11e6-a698-1b05ac77baf9] >>> >>>> Streaming error occurred >>> >>>> java.io.IOException: Broken pipe >>> >>>> at sun.nio.ch.FileChannelImpl.transferTo0(Native Method) >>> >>>> ~[na:1.7.0_79] >>> >>>> at sun.nio.ch.FileChannelImpl.transferToDirectly(Unknown >>> Source) >>> >>>> ~[na:1.7.0_79] >>> >>>> at sun.nio.ch.FileChannelImpl.transferTo(Unknown Source) >>> >>>> ~[na:1.7.0_79] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.compress.CompressedStreamWriter.write(CompressedStreamWriter.java:84) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.messages.OutgoingFileMessage.serialize(OutgoingFileMessage.java:88) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:49) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) >>> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:358) >>> >>>> [apache-cassandra-2.1.14.jar:2.1.14] >>> >>>> at >>> >>>> >>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:330) >>> >>>> [apache-cassandra-2.1.14.jar:2.1.14] >>> >>> >>> >>>>>>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>> >>>>>>>>>>> StreamSession.java:620 - [Stream >>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>> >>>>>>>>>>> Remote peer 192.168.1.140 failed stream session. >>> >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >>> >>>>>>>>>>> StreamSession.java:505 - [Stream >>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>> >>>>>>>>>>> Streaming error occurred >>> >>>>>>>>>>> java.io.IOException: Connection timed out >>> >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native >>> Method) >>> >>>>>>>>>>> ~[na:1.7.0_79] >>> >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>> >>>>>>>>>>> ~[na:1.7.0_79] >>> >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown >>> >>>>>>>>>>> Source) ~[na:1.7.0_79] >>> >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) >>> ~[na:1.7.0_79] >>> >>
Re: Error while rebuilding a node: Stream failed
;> >> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:49) >> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >> >>>> at >> >>>> >> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) >> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >> >>>> at >> >>>> >> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) >> >>>> ~[apache-cassandra-2.1.14.jar:2.1.14] >> >>>> at >> >>>> >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:358) >> >>>> [apache-cassandra-2.1.14.jar:2.1.14] >> >>>> at >> >>>> >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:330) >> >>>> [apache-cassandra-2.1.14.jar:2.1.14] >> >> >> >>>>>>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >> >>>>>>>>>>> StreamSession.java:620 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Remote peer 192.168.1.140 failed stream session. >> >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >> >>>>>>>>>>> StreamSession.java:505 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Streaming error occurred >> >>>>>>>>>>> java.io.IOException: Connection timed out >> >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >> >>>>>>>>>>> ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >> >>>>>>>>>>> ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown >> >>>>>>>>>>> Source) ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) >> ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >> >>>>>>>>>>> ~[na:1.7.0_79] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >> >>>>>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >> >>>>>>>>>>> StreamResultFuture.java:180 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Session with /192.168.1.140 is complete >> >>>>>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >> >>>>>>>>>>> StreamResultFuture.java:207 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Stream failed >> >>>>>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 >> 22:44:58,628 >>
Re: Error while rebuilding a node: Stream failed
org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:330) > >>>> [apache-cassandra-2.1.14.jar:2.1.14] > > > >>>>>>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 > >>>>>>>>>>> StreamSession.java:620 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Remote peer 192.168.1.140 failed stream session. > >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 > >>>>>>>>>>> StreamSession.java:505 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Streaming error occurred > >>>>>>>>>>> java.io.IOException: Connection timed out > >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown > >>>>>>>>>>> Source) ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) > ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) > >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) > >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] > >>>>>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 > >>>>>>>>>>> StreamResultFuture.java:180 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Session with /192.168.1.140 is complete > >>>>>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 > >>>>>>>>>>> StreamResultFuture.java:207 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Stream failed > >>>>>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 > 22:44:58,628 > >>>>>>>>>>> StorageService.java:1075 - Error while rebuilding node > >>>>>>>>>>> org.apache.cassandra.streaming.StreamException: Stream failed > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>&g
Re: Error while rebuilding a node: Stream failed
STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >> >>>>>>>>>>> StreamSession.java:620 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Remote peer 192.168.1.140 failed stream session. >> >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >> >>>>>>>>>>> StreamSession.java:505 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Streaming error occurred >> >>>>>>>>>>> java.io.IOException: Connection timed out >> >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >> >>>>>>>>>>> ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >> >>>>>>>>>>> ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown >> >>>>>>>>>>> Source) ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) >> ~[na:1.7.0_79] >> >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >> >>>>>>>>>>> ~[na:1.7.0_79] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >> >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >> >>>>>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >> >>>>>>>>>>> StreamResultFuture.java:180 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Session with /192.168.1.140 is complete >> >>>>>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >> >>>>>>>>>>> StreamResultFuture.java:207 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] >> >>>>>>>>>>> Stream failed >> >>>>>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 >> 22:44:58,628 >> >>>>>>>>>>> StorageService.java:1075 - Error while rebuilding node >> >>>>>>>>>>> org.apache.cassandra.streaming.StreamException: Stream failed >> >>>>>>>>>>> at >> >>>>>>>>>>> >> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >> >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >> >>>>>>>>>>> ~[guava-16.0.jar:na] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >> >>>>>>>>>>> ~[guava-16.0.jar:na] >> >>>>>>>>>>> at >> >>>>>>>>>>> >> com.google.common.util.concurr
Re: Error while rebuilding a node: Stream failed
;>>>>> java.io.IOException: Connection timed out > >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown > >>>>>>>>>>> Source) ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) > ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) > >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) > >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] > >>>>>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 > >>>>>>>>>>> StreamResultFuture.java:180 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Session with /192.168.1.140 is complete > >>>>>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 > >>>>>>>>>>> StreamResultFuture.java:207 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Stream failed > >>>>>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 > 22:44:58,628 > >>>>>>>>>>> StorageService.java:1075 - Error while rebuilding node > >>>>>>>>>>> org.apache.cassandra.streaming.StreamException: Stream failed > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>>>>>>>>> at > >>>>>>>>>>> > com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) > >>>>>>>>>>> ~[guava-16.0.jar:na] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] > >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 > >>>>>>>>>>> StreamSession.java:505 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] > >>>>>>>>>>> Streaming error occurred > >>>>>>>>>>> java.io.IOException: Broken pipe > >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown > >>>>>>>>>>> Source) ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) > ~[na:1.7.0_79] > >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) > >>>>>>>>>>> ~[na:1.7.0_79] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) > >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) > >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at > >>>>>>>>>>> > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) > >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] > >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] > > > > -- > Eric Evans > john.eric.ev...@gmail.com >
Re: Error while rebuilding a node: Stream failed
0_79] >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>>>>>>> ~[na:1.7.0_79] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>>>>>>>>> StreamResultFuture.java:180 - [Stream >>>>>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>>>>> Session with /192.168.1.140 is complete >>>>>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>>>>>>>>> StreamResultFuture.java:207 - [Stream >>>>>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>>>>> Stream failed >>>>>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>>>>>>>>>> StorageService.java:1075 - Error while rebuilding node >>>>>>>>>>> org.apache.cassandra.streaming.StreamException: Stream failed >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>>>>>>>>>> ~[guava-16.0.jar:na] >>>>>>>>>>> at >>>>>>>>>>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>>>>>>>>>> ~[guava-16.0.jar:na] >>>>>>>>>>> at >>>>>>>>>>> com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) >>>>>>>>>>> ~[guava-16.0.jar:na] >>>>>>>>>>> at >>>>>>>>>>> com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) >>>>>>>>>>> ~[guava-16.0.jar:na] >>>>>>>>>>> at >>>>>>>>>>> com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) >>>>>>>>>>> ~[guava-16.0.jar:na] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] >>>>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 >>>>>>>>>>> StreamSession.java:505 - [Stream >>>>>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>>>>> Streaming error occurred >>>>>>>>>>> java.io.IOException: Broken pipe >>>>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>>>>>>>> ~[na:1.7.0_79] >>>>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>>>>>>>> ~[na:1.7.0_79] >>>>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown >>>>>>>>>>> Source) ~[na:1.7.0_79] >>>>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>>>>>>> ~[na:1.7.0_79] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at >>>>>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>>>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] -- Eric Evans john.eric.ev...@gmail.com
Re: Error while rebuilding a node: Stream failed
t;>>> >>>>>>> We are addressing this on >>>>>>> https://issues.apache.org/jira/browse/CASSANDRA-11839. >>>>>>> >>>>>>> 2016-05-25 16:42 GMT-03:00 George Sigletos <sigle...@textkernel.nl>: >>>>>>> >>>>>>>> Hello again, >>>>>>>> >>>>>>>> Here is the error message from the source >>>>>>>> >>>>>>>> INFO [STREAM-IN-/172.31.22.104] 2016-05-25 00:44:57,275 >>>>>>>> StreamResultFuture.java:180 - [Stream >>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /172.31.22.104 >>>>>>>> is complete >>>>>>>> WARN [STREAM-IN-/172.31.22.104] 2016-05-25 00:44:57,276 >>>>>>>> StreamResultFuture.java:207 - [Stream >>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>>>>>>> ERROR [STREAM-OUT-/172.31.22.104] 2016-05-25 00:44:57,353 >>>>>>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>> Streaming error occurred >>>>>>>> java.lang.AssertionError: Memory was freed >>>>>>>> at >>>>>>>> org.apache.cassandra.io.util.SafeMemory.checkBounds(SafeMemory.java:97) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.io.util.Memory.getLong(Memory.java:249) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.io.compress.CompressionMetadata.getTotalSizeForSections(CompressionMetadata.java:247) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.messages.FileMessageHeader.size(FileMessageHeader.java:112) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.StreamSession.fileSent(StreamSession.java:546) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:50) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>>>> >>>>>>>> On Wed, May 25, 2016 at 8:49 PM, Paulo Motta < >>>>>>>> pauloricard...@gmail.com> wrote: >>>>>>>> >>>>>>>>> This is the log of the destination/rebuilding node, you need to >>>>>>>>> check what is the error message on the stream source node >>>>>>>>> (192.168.1.140). >>>>>>>>> >>>>>>>>> >>>>>>>>> 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl >>>>>>>>> >: >>>>>>>>> >>>>>>>>>> Hello, >>>>>>>>>> >>>>>>>>>> Here is additional stack trace from system.log: >>>>>>>>>> >>>>>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>>>>>>>
Re: Error while rebuilding a node: Stream failed
il.SafeMemory.checkBounds(SafeMemory.java:97) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.io.util.Memory.getLong(Memory.java:249) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.io.compress.CompressionMetadata.getTotalSizeForSections(CompressionMetadata.java:247) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.messages.FileMessageHeader.size(FileMessageHeader.java:112) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.StreamSession.fileSent(StreamSession.java:546) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:50) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>>> >>>>>>> On Wed, May 25, 2016 at 8:49 PM, Paulo Motta < >>>>>>> pauloricard...@gmail.com> wrote: >>>>>>> >>>>>>>> This is the log of the destination/rebuilding node, you need to >>>>>>>> check what is the error message on the stream source node >>>>>>>> (192.168.1.140). >>>>>>>> >>>>>>>> >>>>>>>> 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl> >>>>>>>> : >>>>>>>> >>>>>>>>> Hello, >>>>>>>>> >>>>>>>>> Here is additional stack trace from system.log: >>>>>>>>> >>>>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>>>>>>>> StreamSession.java:620 - [Stream >>>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>>> Remote peer 192.168.1.140 failed stream session. >>>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >>>>>>>>> StreamSession.java:505 - [Stream >>>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>>> Streaming error occurred >>>>>>>>> java.io.IOException: Connection timed out >>>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>>>>>> ~[na:1.7.0_79] >>>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>>>>>> ~[na:1.7.0_79] >>>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>>>>>>> ~[na:1.7.0_79] >>>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>>>>> ~[na:1.7.0_79] >>>>>>>>> at >>>>>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>>> at >>>>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>&g
Re: Error while rebuilding a node: Stream failed
rialize(OutgoingFileMessage.java:41) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>> >>>>>> On Wed, May 25, 2016 at 8:49 PM, Paulo Motta < >>>>>> pauloricard...@gmail.com> wrote: >>>>>> >>>>>>> This is the log of the destination/rebuilding node, you need to >>>>>>> check what is the error message on the stream source node >>>>>>> (192.168.1.140). >>>>>>> >>>>>>> >>>>>>> 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl>: >>>>>>> >>>>>>>> Hello, >>>>>>>> >>>>>>>> Here is additional stack trace from system.log: >>>>>>>> >>>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>>>>>>> StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>> Remote peer 192.168.1.140 failed stream session. >>>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >>>>>>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>>> Streaming error occurred >>>>>>>> java.io.IOException: Connection timed out >>>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>>>>> ~[na:1.7.0_79] >>>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>>>>> ~[na:1.7.0_79] >>>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>>>>>> ~[na:1.7.0_79] >>>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>>>> ~[na:1.7.0_79] >>>>>>>> at >>>>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>>>>>> StreamResultFuture.java:180 - [Stream >>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 >>>>>>>> is complete >>>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>>>>>> StreamResultFuture.java:207 - [Stream >>>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>>>>>>> StorageService.java:1075 - Error while rebuilding node >>>>>>>> org.apache.cassandra.streaming.StreamException: Stream failed >>>>>>>> at >>>>>>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifie
Re: Error while rebuilding a node: Stream failed
gt; >>>>>>> Hello, >>>>>>> >>>>>>> Here is additional stack trace from system.log: >>>>>>> >>>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>>>>>> StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>> Remote peer 192.168.1.140 failed stream session. >>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >>>>>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>> Streaming error occurred >>>>>>> java.io.IOException: Connection timed out >>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>>>> ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>>>> ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>>>>> ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>>> ~[na:1.7.0_79] >>>>>>> at >>>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>>>>> StreamResultFuture.java:180 - [Stream >>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 >>>>>>> is complete >>>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>>>>> StreamResultFuture.java:207 - [Stream >>>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>>>>>> StorageService.java:1075 - Error while rebuilding node >>>>>>> org.apache.cassandra.streaming.StreamException: Stream failed >>>>>>> at >>>>>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>>>>>> ~[guava-16.0.jar:na] >>>>>>> at >>>>>>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>>>>>> ~[guava-16.0.jar:na] >>>>>>> at >>>>>>> com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) >>>>>>> ~[guava-16.0.jar:na] >>>>>>> at >>>>>>> com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) >>>>>>> ~[guava-16.0.jar:na] >>>>>>> at >>>>>>> com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) >>>>>>> ~[guava-16.0.jar:na] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] >>>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 >>>>>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>>> Streaming error occurred >>>>>>> java.io.IOException: Broken pipe >>>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>>>> ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>>>> ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>>>>> ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>>> ~[na:1.7.0_79] >>>>>>> at >>>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at >>>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>>> >>>>>>> >>>>>>> On Wed, May 25, 2016 at 5:23 PM, Paulo Motta < >>>>>>> pauloricard...@gmail.com> wrote: >>>>>>> >>>>>>>> The stack trace from the rebuild command not show the root cause of >>>>>>>> the rebuild stream error. Can you check the system.log for ERROR logs >>>>>>>> during streaming and paste here? >>>>>>>> >>>>>>> >>>>>>> >>>>>> >>>>> >>>> >>> >> >
Re: Error while rebuilding a node: Stream failed
> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>> >>>> On Wed, May 25, 2016 at 8:49 PM, Paulo Motta <pauloricard...@gmail.com> >>>> wrote: >>>> >>>>> This is the log of the destination/rebuilding node, you need to check >>>>> what is the error message on the stream source node (192.168.1.140). >>>>> >>>>> >>>>> 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl>: >>>>> >>>>>> Hello, >>>>>> >>>>>> Here is additional stack trace from system.log: >>>>>> >>>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>>>>> StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>> Remote peer 192.168.1.140 failed stream session. >>>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >>>>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>>> Streaming error occurred >>>>>> java.io.IOException: Connection timed out >>>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>>> ~[na:1.7.0_79] >>>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>>> ~[na:1.7.0_79] >>>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>>>> ~[na:1.7.0_79] >>>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>>> ~[na:1.7.0_79] >>>>>> at >>>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>>>> StreamResultFuture.java:180 - [Stream >>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 >>>>>> is complete >>>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>>>> StreamResultFuture.java:207 - [Stream >>>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>>>>> StorageService.java:1075 - Error while rebuilding node >>>>>> org.apache.cassandra.streaming.StreamException: Stream failed >>>>>> at >>>>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>>> at >>>>>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>>>>> ~[guava-16.0.jar:na] >>>>>> at >>>>>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>>>>> ~[guava-16.0.jar:na] >>>>>> at >>>>>> com.google.common.util.concurrent.ExecutionList.executeL
Re: Error while rebuilding a node: Stream failed
o check >>>> what is the error message on the stream source node (192.168.1.140). >>>> >>>> >>>> 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl>: >>>> >>>>> Hello, >>>>> >>>>> Here is additional stack trace from system.log: >>>>> >>>>> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >>>>> StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>> Remote peer 192.168.1.140 failed stream session. >>>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >>>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>>> Streaming error occurred >>>>> java.io.IOException: Connection timed out >>>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>>> ~[na:1.7.0_79] >>>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>>> ~[na:1.7.0_79] >>>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>>> ~[na:1.7.0_79] >>>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>>> ~[na:1.7.0_79] >>>>> at >>>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>>> StreamResultFuture.java:180 - [Stream >>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 is >>>>> complete >>>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>>> StreamResultFuture.java:207 - [Stream >>>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>>>> StorageService.java:1075 - Error while rebuilding node >>>>> org.apache.cassandra.streaming.StreamException: Stream failed >>>>> at >>>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>>>> ~[guava-16.0.jar:na] >>>>> at >>>>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>>>> ~[guava-16.0.jar:na] >>>>> at >>>>> com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) >>>>> ~[guava-16.0.jar:na] >>>>> at >>>>> com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) >>>>> ~[guava-16.0.jar:na] >>>>> at >>>>> com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) >>>>> ~[guava-16.0.jar:na] >>>>> at >>>>> org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) >>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) >>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) >>>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>>> at >>>>> org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) >>>>> ~[apac
Re: Error while rebuilding a node: Stream failed
nnel.java:48) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>>> StreamResultFuture.java:180 - [Stream >>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 is >>>> complete >>>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>>> StreamResultFuture.java:207 - [Stream >>>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>>> StorageService.java:1075 - Error while rebuilding node >>>> org.apache.cassandra.streaming.StreamException: Stream failed >>>> at >>>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>>> ~[guava-16.0.jar:na] >>>> at >>>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>>> ~[guava-16.0.jar:na] >>>> at >>>> com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) >>>> ~[guava-16.0.jar:na] >>>> at >>>> com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) >>>> ~[guava-16.0.jar:na] >>>> at >>>> com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) >>>> ~[guava-16.0.jar:na] >>>> at >>>> org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] >>>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 >>>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>>> Streaming error occurred >>>> java.io.IOException: Broken pipe >>>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>>> ~[na:1.7.0_79] >>>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>>> ~[na:1.7.0_79] >>>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>>> ~[na:1.7.0_79] >>>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>>> ~[na:1.7.0_79] >>>> at >>>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> at >>>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>>> [apache-cassandra-2.1.13.jar:2.1.13] >>>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>>> >>>> >>>> On Wed, May 25, 2016 at 5:23 PM, Paulo Motta <pauloricard...@gmail.com> >>>> wrote: >>>> >>>>> The stack trace from the rebuild command not show the root cause of >>>>> the rebuild stream error. Can you check the system.log for ERROR logs >>>>> during streaming and paste here? >>>>> >>>> >>>> >>> >> >
Re: Error while rebuilding a node: Stream failed
gt; [apache-cassandra-2.1.13.jar:2.1.13] >>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >>> StreamResultFuture.java:180 - [Stream >>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 is >>> complete >>> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >>> StreamResultFuture.java:207 - [Stream >>> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >>> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >>> StorageService.java:1075 - Error while rebuilding node >>> org.apache.cassandra.streaming.StreamException: Stream failed >>> at >>> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) >>> ~[guava-16.0.jar:na] >>> at >>> com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) >>> ~[guava-16.0.jar:na] >>> at >>> com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) >>> ~[guava-16.0.jar:na] >>> at >>> com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) >>> ~[guava-16.0.jar:na] >>> at >>> com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) >>> ~[guava-16.0.jar:na] >>> at >>> org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] >>> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 >>> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >>> Streaming error occurred >>> java.io.IOException: Broken pipe >>> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >>> ~[na:1.7.0_79] >>> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >>> ~[na:1.7.0_79] >>> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >>> ~[na:1.7.0_79] >>> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >>> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >>> ~[na:1.7.0_79] >>> at >>> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >>> ~[apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >>> [apache-cassandra-2.1.13.jar:2.1.13] >>> at >>> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) >>> [apache-cassandra-2.1.13.jar:2.1.13] >>> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >>> >>> >>> On Wed, May 25, 2016 at 5:23 PM, Paulo Motta <pauloricard...@gmail.com> >>> wrote: >>> >>>> The stack trace from the rebuild command not show the root cause of the >>>> rebuild stream error. Can you check the system.log for ERROR logs during >>>> streaming and paste here? >>>> >>> >>> >> >
Re: Error while rebuilding a node: Stream failed
Hello again, Here is the error message from the source INFO [STREAM-IN-/172.31.22.104] 2016-05-25 00:44:57,275 StreamResultFuture.java:180 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /172.31.22.104 is complete WARN [STREAM-IN-/172.31.22.104] 2016-05-25 00:44:57,276 StreamResultFuture.java:207 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed ERROR [STREAM-OUT-/172.31.22.104] 2016-05-25 00:44:57,353 StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Streaming error occurred java.lang.AssertionError: Memory was freed at org.apache.cassandra.io.util.SafeMemory.checkBounds(SafeMemory.java:97) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.io.util.Memory.getLong(Memory.java:249) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.io.compress.CompressionMetadata.getTotalSizeForSections(CompressionMetadata.java:247) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.messages.FileMessageHeader.size(FileMessageHeader.java:112) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.StreamSession.fileSent(StreamSession.java:546) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:50) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.messages.OutgoingFileMessage$1.serialize(OutgoingFileMessage.java:41) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:45) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) ~[apache-cassandra-2.1.13.jar:2.1.13] at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] On Wed, May 25, 2016 at 8:49 PM, Paulo Motta <pauloricard...@gmail.com> wrote: > This is the log of the destination/rebuilding node, you need to check what > is the error message on the stream source node (192.168.1.140). > > > 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl>: > >> Hello, >> >> Here is additional stack trace from system.log: >> >> ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 >> StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >> Remote peer 192.168.1.140 failed stream session. >> ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 >> StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] >> Streaming error occurred >> java.io.IOException: Connection timed out >> at sun.nio.ch.FileDispatcherImpl.write0(Native Method) >> ~[na:1.7.0_79] >> at sun.nio.ch.SocketDispatcher.write(Unknown Source) >> ~[na:1.7.0_79] >> at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) >> ~[na:1.7.0_79] >> at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] >> at sun.nio.ch.SocketChannelImpl.write(Unknown Source) >> ~[na:1.7.0_79] >> at >> org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) >> ~[apache-cassandra-2.1.13.jar:2.1.13] >> at >> org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) >> ~[apache-cassandra-2.1.13.jar:2.1.13] >> at >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) >> [apache-cassandra-2.1.13.jar:2.1.13] >> at >> org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) >> [apache-cassandra-2.1.13.jar:2.1.13] >> at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] >> INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 >> StreamResultFuture.java:180 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 is >> complete >> WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 >> StreamResultFuture.java:207 - [Stream >> #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed >> ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 >> StorageService.java:1075 - Error while rebuilding node >> org.apache.cassandra.streaming.StreamException: Stream failed >> at >> org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) >> ~[apache-cassandra-2.1.13.jar:2.1.13] >> at >> com.google.common.util.concurrent.Futures$4.run
Re: Error while rebuilding a node: Stream failed
This is the log of the destination/rebuilding node, you need to check what is the error message on the stream source node (192.168.1.140). 2016-05-25 15:22 GMT-03:00 George Sigletos <sigle...@textkernel.nl>: > Hello, > > Here is additional stack trace from system.log: > > ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 > StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] > Remote peer 192.168.1.140 failed stream session. > ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 > StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] > Streaming error occurred > java.io.IOException: Connection timed out > at sun.nio.ch.FileDispatcherImpl.write0(Native Method) > ~[na:1.7.0_79] > at sun.nio.ch.SocketDispatcher.write(Unknown Source) ~[na:1.7.0_79] > at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) > ~[na:1.7.0_79] > at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] > at sun.nio.ch.SocketChannelImpl.write(Unknown Source) > ~[na:1.7.0_79] > at > org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) > [apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) > [apache-cassandra-2.1.13.jar:2.1.13] > at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] > INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 > StreamResultFuture.java:180 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 is > complete > WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 > StreamResultFuture.java:207 - [Stream > #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed > ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 > StorageService.java:1075 - Error while rebuilding node > org.apache.cassandra.streaming.StreamException: Stream failed > at > org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) > ~[guava-16.0.jar:na] > at > com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) > ~[guava-16.0.jar:na] > at > com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) > ~[guava-16.0.jar:na] > at > com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) > ~[guava-16.0.jar:na] > at > com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) > ~[guava-16.0.jar:na] > at > org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at > org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) > ~[apache-cassandra-2.1.13.jar:2.1.13] > at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] > ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 > StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] > Streaming error occurred > java.io.IOException: Broken pipe > at sun.nio.ch.FileDispatcherImpl.write0(Native Method) > ~[na:1.7.0_79] > at sun.nio.ch.SocketDispatcher.write(Unknown Source) ~[na:1.7.0_79] > at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) > ~[na:1.7.0_79] > at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] > at sun.nio.ch.SocketChannelImpl.write(Unknown Source) > ~[na:1.7.0_79] > at > org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) > ~[apache-cassandr
Re: Error while rebuilding a node: Stream failed
Hello, Here is additional stack trace from system.log: ERROR [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:57,704 StreamSession.java:620 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Remote peer 192.168.1.140 failed stream session. ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:57,705 StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Streaming error occurred java.io.IOException: Connection timed out at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[na:1.7.0_79] at sun.nio.ch.SocketDispatcher.write(Unknown Source) ~[na:1.7.0_79] at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) ~[na:1.7.0_79] at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] at sun.nio.ch.SocketChannelImpl.write(Unknown Source) ~[na:1.7.0_79] at org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) [apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:323) [apache-cassandra-2.1.13.jar:2.1.13] at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] INFO [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,625 StreamResultFuture.java:180 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Session with /192.168.1.140 is complete WARN [STREAM-IN-/192.168.1.140] 2016-05-24 22:44:58,627 StreamResultFuture.java:207 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Stream failed ERROR [RMI TCP Connection(24)-127.0.0.1] 2016-05-24 22:44:58,628 StorageService.java:1075 - Error while rebuilding node org.apache.cassandra.streaming.StreamException: Stream failed at org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85) ~[apache-cassandra-2.1.13.jar:2.1.13] at com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) ~[guava-16.0.jar:na] at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) ~[guava-16.0.jar:na] at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156) ~[guava-16.0.jar:na] at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145) ~[guava-16.0.jar:na] at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202) ~[guava-16.0.jar:na] at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:208) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:184) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:415) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.StreamSession.sessionFailed(StreamSession.java:621) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.StreamSession.messageReceived(StreamSession.java:475) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:256) ~[apache-cassandra-2.1.13.jar:2.1.13] at java.lang.Thread.run(Unknown Source) ~[na:1.7.0_79] ERROR [STREAM-OUT-/192.168.1.140] 2016-05-24 22:44:58,629 StreamSession.java:505 - [Stream #2c290460-20d4-11e6-930f-1b05ac77baf9] Streaming error occurred java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[na:1.7.0_79] at sun.nio.ch.SocketDispatcher.write(Unknown Source) ~[na:1.7.0_79] at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) ~[na:1.7.0_79] at sun.nio.ch.IOUtil.write(Unknown Source) ~[na:1.7.0_79] at sun.nio.ch.SocketChannelImpl.write(Unknown Source) ~[na:1.7.0_79] at org.apache.cassandra.io.util.DataOutputStreamAndChannel.write(DataOutputStreamAndChannel.java:48) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.messages.StreamMessage.serialize(StreamMessage.java:44) ~[apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.sendMessage(ConnectionHandler.java:351) [apache-cassandra-2.1.13.jar:2.1.13] at org.apache.cassandra.streaming.ConnectionHandler$OutgoingMessageHandler.run(ConnectionHandler.java:331) [apache-cassandra-2.1.13.jar:2.1.13] at java.lang.Thread.run(Unknown Source) [na:1.7.0_79] On Wed, May 25, 2016 at 5:23 PM, Paulo Motta <pauloricard...@gmail.com> wrote: > The stack trace from the rebuild command not show the r
Re: Error while rebuilding a node: Stream failed
The stack trace from the rebuild command not show the root cause of the rebuild stream error. Can you check the system.log for ERROR logs during streaming and paste here?
Re: Error while rebuilding a node: Stream failed
Hi Mike, Yes I am using NetworkTopologyStrategy. I checked cassandra-rackdc.properties on the new node: dc=DCamazon-1 rack=RACamazon-1 I also checked the jira link you sent me. My network topology seems correct: I have 4 nodes in DC1 and 1 node in DCamazon-1 and I can verify that when running "nodetool status". Now I am running a full repair on the amazon node. I have given up rebuilding Kind regards, George On Wed, May 25, 2016 at 8:50 AM, Mike Yeap <wkk1...@gmail.com> wrote: > Hi George, are you using NetworkTopologyStrategy as the replication > strategy for your keyspace? If yes, can you check the > cassandra-rackdc.properties of this new node? > > https://issues.apache.org/jira/browse/CASSANDRA-8279 > > > Regards, > Mike Yeap > > On Wed, May 25, 2016 at 2:31 PM, George Sigletos <sigle...@textkernel.nl> > wrote: > >> I am getting this error repeatedly while I am trying to add a new DC >> consisting of one node in AWS to my existing cluster. I have tried 5 times >> already. Running Cassandra 2.1.13 >> >> I have also set: >> streaming_socket_timeout_in_ms: 360 >> in all of my nodes >> >> Does anybody have any idea how this can be fixed? Thanks in advance >> >> Kind regards, >> George >> >> P.S. >> The complete stack trace: >> -- StackTrace -- >> java.lang.RuntimeException: Error while rebuilding node: Stream failed >> at >> org.apache.cassandra.service.StorageService.rebuild(StorageService.java:1076) >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) >> at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) >> at java.lang.reflect.Method.invoke(Unknown Source) >> at sun.reflect.misc.Trampoline.invoke(Unknown Source) >> at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) >> at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) >> at java.lang.reflect.Method.invoke(Unknown Source) >> at sun.reflect.misc.MethodUtil.invoke(Unknown Source) >> at >> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(Unknown Source) >> at >> com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(Unknown Source) >> at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(Unknown >> Source) >> at com.sun.jmx.mbeanserver.PerInterface.invoke(Unknown Source) >> at com.sun.jmx.mbeanserver.MBeanSupport.invoke(Unknown Source) >> at >> com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(Unknown Source) >> at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(Unknown Source) >> at >> javax.management.remote.rmi.RMIConnectionImpl.doOperation(Unknown Source) >> at >> javax.management.remote.rmi.RMIConnectionImpl.access$300(Unknown Source) >> at >> javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(Unknown >> Source) >> at >> javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(Unknown >> Source) >> at javax.management.remote.rmi.RMIConnectionImpl.invoke(Unknown >> Source) >> at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) >> at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) >> at java.lang.reflect.Method.invoke(Unknown Source) >> at sun.rmi.server.UnicastServerRef.dispatch(Unknown Source) >> at sun.rmi.transport.Transport$2.run(Unknown Source) >> at sun.rmi.transport.Transport$2.run(Unknown Source) >> at java.security.AccessController.doPrivileged(Native Method) >> at sun.rmi.transport.Transport.serviceCall(Unknown Source) >> at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown >> Source) >> at >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) >> at >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.access$400(Unknown >> Source) >> at >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$1.run(Unknown Source) >> at >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$1.run(Unknown Source) >> at java.security.AccessController.doPrivileged(Native Method) >> at >> sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) >> at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown >> Source) >> at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown >> Source) >> at java.lang.Thread.run(Unknown Source) >> > >
Re: Error while rebuilding a node: Stream failed
Hi George, are you using NetworkTopologyStrategy as the replication strategy for your keyspace? If yes, can you check the cassandra-rackdc.properties of this new node? https://issues.apache.org/jira/browse/CASSANDRA-8279 Regards, Mike Yeap On Wed, May 25, 2016 at 2:31 PM, George Sigletos <sigle...@textkernel.nl> wrote: > I am getting this error repeatedly while I am trying to add a new DC > consisting of one node in AWS to my existing cluster. I have tried 5 times > already. Running Cassandra 2.1.13 > > I have also set: > streaming_socket_timeout_in_ms: 360 > in all of my nodes > > Does anybody have any idea how this can be fixed? Thanks in advance > > Kind regards, > George > > P.S. > The complete stack trace: > -- StackTrace -- > java.lang.RuntimeException: Error while rebuilding node: Stream failed > at > org.apache.cassandra.service.StorageService.rebuild(StorageService.java:1076) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) > at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) > at java.lang.reflect.Method.invoke(Unknown Source) > at sun.reflect.misc.Trampoline.invoke(Unknown Source) > at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) > at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) > at java.lang.reflect.Method.invoke(Unknown Source) > at sun.reflect.misc.MethodUtil.invoke(Unknown Source) > at > com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(Unknown Source) > at > com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(Unknown Source) > at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(Unknown > Source) > at com.sun.jmx.mbeanserver.PerInterface.invoke(Unknown Source) > at com.sun.jmx.mbeanserver.MBeanSupport.invoke(Unknown Source) > at > com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(Unknown Source) > at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(Unknown Source) > at > javax.management.remote.rmi.RMIConnectionImpl.doOperation(Unknown Source) > at > javax.management.remote.rmi.RMIConnectionImpl.access$300(Unknown Source) > at > javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(Unknown > Source) > at > javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(Unknown > Source) > at javax.management.remote.rmi.RMIConnectionImpl.invoke(Unknown > Source) > at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) > at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) > at java.lang.reflect.Method.invoke(Unknown Source) > at sun.rmi.server.UnicastServerRef.dispatch(Unknown Source) > at sun.rmi.transport.Transport$2.run(Unknown Source) > at sun.rmi.transport.Transport$2.run(Unknown Source) > at java.security.AccessController.doPrivileged(Native Method) > at sun.rmi.transport.Transport.serviceCall(Unknown Source) > at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown > Source) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.access$400(Unknown > Source) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$1.run(Unknown Source) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$1.run(Unknown Source) > at java.security.AccessController.doPrivileged(Native Method) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) > at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown > Source) > at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown > Source) > at java.lang.Thread.run(Unknown Source) >
Error while rebuilding a node: Stream failed
I am getting this error repeatedly while I am trying to add a new DC consisting of one node in AWS to my existing cluster. I have tried 5 times already. Running Cassandra 2.1.13 I have also set: streaming_socket_timeout_in_ms: 360 in all of my nodes Does anybody have any idea how this can be fixed? Thanks in advance Kind regards, George P.S. The complete stack trace: -- StackTrace -- java.lang.RuntimeException: Error while rebuilding node: Stream failed at org.apache.cassandra.service.StorageService.rebuild(StorageService.java:1076) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.reflect.misc.Trampoline.invoke(Unknown Source) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.reflect.misc.MethodUtil.invoke(Unknown Source) at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(Unknown Source) at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(Unknown Source) at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(Unknown Source) at com.sun.jmx.mbeanserver.PerInterface.invoke(Unknown Source) at com.sun.jmx.mbeanserver.MBeanSupport.invoke(Unknown Source) at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(Unknown Source) at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.doOperation(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.access$300(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(Unknown Source) at javax.management.remote.rmi.RMIConnectionImpl.invoke(Unknown Source) at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at sun.rmi.server.UnicastServerRef.dispatch(Unknown Source) at sun.rmi.transport.Transport$2.run(Unknown Source) at sun.rmi.transport.Transport$2.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Unknown Source) at sun.rmi.transport.tcp.TCPTransport.handleMessages(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.access$400(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$1.run(Unknown Source) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$1.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source)
