[jira] [Updated] (CASSANDRA-11826) Repairs failing for some of bigger tables
[ https://issues.apache.org/jira/browse/CASSANDRA-11826?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] vin01 updated CASSANDRA-11826: -- Description: Architecture for the same :- Cassandra version 2.2.4 3 Nodes - Node-1, Node-2, Node-3, repair started on Node-1 and following logs are also from Node-1 itself. Node-2 and Node-3 are in same data-center, and Node-1 is in a different data-center. Value of streaming_socket_timeout_in_ms is set to 1 Hour which is default for version 2.2.4 Logs for the same :- WARN [RepairJobTask:5] 2016-05-18 02:25:31,137 RepairJob.java:162 - [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd] TABLE_NAME sync failed ERROR [RepairJobTask:5] 2016-05-18 02:25:31,138 RepairSession.java:290 - [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd] Session completed with the following error org.apache.cassandra.exceptions.RepairException: [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd on KEYSPACE_NAME/TABLE_NAME, (-7892648649079895028,-7818964903881740266]] Sync failed between /Node-3 and /Node-2 at org.apache.cassandra.repair.RemoteSyncTask.syncComplete(RemoteSyncTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairSession.syncComplete(RepairSession.java:211) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:415) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:163) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72] ERROR [RepairJobTask:5] 2016-05-18 02:25:31,138 RepairRunnable.java:243 - Repair session 5a855825-1cb7-11e6-9f5f-b717b380ffdd for range (-7892648649079895028,-7818964903881740266] failed with error [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd on KEYSPACE_NA$ org.apache.cassandra.exceptions.RepairException: [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd on KEYSPACE_NAME/TABLE_NAME, (-7892648649079895028,-7818964903881740266]] Sync failed between /Node-3 and /Node-2 at org.apache.cassandra.repair.RemoteSyncTask.syncComplete(RemoteSyncTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairSession.syncComplete(RepairSession.java:211) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:415) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:163) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72] ERROR [STREAM-IN-/Node-2] 2016-05-18 02:25:33,296 StreamSession.java:524 - [Stream #efe521b0-1cb8-11e6-9f5f-b717b380ffdd] Streaming error occurred java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) ~[na:1.8.0_72] at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) ~[na:1.8.0_72] at java.net.SocketInputStream.read(SocketInputStream.java:170) ~[na:1.8.0_72] at java.net.SocketInputStream.read(SocketInputStream.java:141) ~[na:1.8.0_72] at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) ~[na:1.8.0_72] at sun.security.ssl.InputRecord.read(InputRecord.java:503) ~[na:1.8.0_72] at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973) ~[na:1.8.0_72] at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930) ~[na:1.8.0_72] at sun.security.ssl.AppInputStream.read(AppInputStream.java:105) ~[na:1.8.0_72] at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) ~[na:1.8.0_72] at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:53) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:261) ~[apache-cassandra-2.2.4.jar:2.2.4] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72] INFO [STREAM-IN-/Node-2] 2016-05-18 02:25:33,296 StreamResultFuture.java:182 - [
[jira] [Updated] (CASSANDRA-11826) Repairs failing for some of bigger tables
[ https://issues.apache.org/jira/browse/CASSANDRA-11826?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] vin01 updated CASSANDRA-11826: -- Description: Architecture for the same :- Cassandra version 2.2.4 3 Nodes - Node-1, Node-2, Node-3, repair started on Node-1 and following logs are also from Node-1 itself. Node-2 and Node-3 are in same data-center, and Node-1 is in a different data-center. Value of streaming_socket_timeout_in_ms is set to 1 Hour which is default for version 2.2.4. Repair succeeded for some of table but failed for one of tables which is quite huge, i looked at sizes of .db files in data directory and there are files up to 1.5 GB for that table. 1.4G la-59543-big-Data.db Logs for the same when repair failed :- WARN [RepairJobTask:5] 2016-05-18 02:25:31,137 RepairJob.java:162 - [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd] TABLE_NAME sync failed ERROR [RepairJobTask:5] 2016-05-18 02:25:31,138 RepairSession.java:290 - [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd] Session completed with the following error org.apache.cassandra.exceptions.RepairException: [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd on KEYSPACE_NAME/TABLE_NAME, (-7892648649079895028,-7818964903881740266]] Sync failed between /Node-3 and /Node-2 at org.apache.cassandra.repair.RemoteSyncTask.syncComplete(RemoteSyncTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairSession.syncComplete(RepairSession.java:211) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:415) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:163) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72] ERROR [RepairJobTask:5] 2016-05-18 02:25:31,138 RepairRunnable.java:243 - Repair session 5a855825-1cb7-11e6-9f5f-b717b380ffdd for range (-7892648649079895028,-7818964903881740266] failed with error [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd on KEYSPACE_NA$ org.apache.cassandra.exceptions.RepairException: [repair #5a855825-1cb7-11e6-9f5f-b717b380ffdd on KEYSPACE_NAME/TABLE_NAME, (-7892648649079895028,-7818964903881740266]] Sync failed between /Node-3 and /Node-2 at org.apache.cassandra.repair.RemoteSyncTask.syncComplete(RemoteSyncTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairSession.syncComplete(RepairSession.java:211) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:415) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:163) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) ~[apache-cassandra-2.2.4.jar:2.2.4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_72] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_72] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_72] ERROR [STREAM-IN-/Node-2] 2016-05-18 02:25:33,296 StreamSession.java:524 - [Stream #efe521b0-1cb8-11e6-9f5f-b717b380ffdd] Streaming error occurred java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) ~[na:1.8.0_72] at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) ~[na:1.8.0_72] at java.net.SocketInputStream.read(SocketInputStream.java:170) ~[na:1.8.0_72] at java.net.SocketInputStream.read(SocketInputStream.java:141) ~[na:1.8.0_72] at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) ~[na:1.8.0_72] at sun.security.ssl.InputRecord.read(InputRecord.java:503) ~[na:1.8.0_72] at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973) ~[na:1.8.0_72] at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930) ~[na:1.8.0_72] at sun.security.ssl.AppInputStream.read(AppInputStream.java:105) ~[na:1.8.0_72] at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) ~[na:1.8.0_72] at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:53) ~[apache-cassandra-2.2.4.jar:2.2.4] at org.apache.cassandra.streaming.ConnectionHandler$Inc
