[ https://issues.apache.org/jira/browse/CASSANDRA-19209?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Kapil Shewate updated CASSANDRA-19209: -------------------------------------- Description: On Cassandra 4.0.7 we are seeing Merkle tree repair errors , we are using cassandra-reaper to do continuous repairs, below is the exception ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,131 RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [WARN] [RepairJobTask:3] 2023-11-05 13:49:58,132 RepairJob.java:177 - repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.payloads_by_bucket sync failed [WARN] [RepairJobTask:8] 2023-11-05 13:49:58,132 RepairJob.java:177 - repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.segments sync failed [WARN] [RepairJobTask:7] 2023-11-05 13:49:58,133 RepairJob.java:177 - repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.replication_queue sync failed [WARN] [RepairJobTask:11] 2023-11-05 13:49:58,133 RepairJob.java:177 - repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.payload_register sync failed [ERROR] [Repair#240941:1] 2023-11-05 13:49:58,134 RepairSession.java:321 - repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 Session completed with the following error org.apache.cassandra.exceptions.RepairException: [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from /<IP>:7000: UNKNOWN at org.apache.cassandra.repair.messages.RepairMessage$1.onFailure(RepairMessage.java:81) at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45) at org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:432) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:522) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:119) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [WARN] [RepairJobTask:3] 2023-11-05 13:49:58,134 RepairJob.java:177 - repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.replication_queue_ptr sync failed [ERROR] [Repair#240941:1] 2023-11-05 13:49:58,134 RepairRunnable.java:178 - Repair 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 failed: java.lang.RuntimeException: Repair session 7fc76c30-7c14-11ee-a6b1-09b8491c94a6 for range [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]] failed with error [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from /<IP>:7000: UNKNOWN at org.apache.cassandra.repair.RepairRunnable$RepairSessionCallback.onFailure(RepairRunnable.java:698) at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1056) at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:30) at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1138) at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:958) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:748) at org.apache.cassandra.repair.RepairSession.forceShutdown(RepairSession.java:342) at org.apache.cassandra.repair.RepairSession$1.onFailure(RepairSession.java:323) at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1056) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) Caused by: org.apache.cassandra.exceptions.RepairException: [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from /<IP>:7000: UNKNOWN at org.apache.cassandra.repair.messages.RepairMessage$1.onFailure(RepairMessage.java:81) at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45) at org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:432) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:522) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:119) ... 2 common frames omitted [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,136 Validator.java:237 - Failed creating a merkle tree for [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1cae277e_6e2e_46d2_bffa_983557f6b155, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1cae277e_6e2e_46d2_bffa_983557f6b155, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]], /<IP>:7000 (see log for details) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 ValidationManager.java:173 - Validation failed. java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 CassandraDaemon.java:581 - Exception in thread Thread[ValidationExecutor:66,1,main] java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 Validator.java:237 - Failed creating a merkle tree for [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1a116929_41ec_4be6_b561_0d42dad8caf7, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1a116929_41ec_4be6_b561_0d42dad8caf7, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]], /<IP>:7000 (see log for details) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 ValidationManager.java:173 - Validation failed. java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 CassandraDaemon.java:581 - Exception in thread Thread[ValidationExecutor:66,1,main] java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) was: On Cassandra 4.0.7 we are seeing Merkle tree repair errors , we are using cassandra-reaper to do continuous repairs, below is the exception ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,131 RepairMessage.java:78 - [#7fbb5e40-7c14-11ee-a6b1-09b8491c94a6] VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - [#7fbb5e40-7c14-11ee-a6b1-09b8491c94a6] VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - [#7fbb5e40-7c14-11ee-a6b1-09b8491c94a6] VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - [#7fbb5e40-7c14-11ee-a6b1-09b8491c94a6] VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 RepairMessage.java:78 - [#7fbb5e40-7c14-11ee-a6b1-09b8491c94a6] VALIDATION_REQ failed on /<IP>:7000: UNKNOWN [WARN] [RepairJobTask:3] 2023-11-05 13:49:58,132 RepairJob.java:177 - [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6] replication.payloads_by_bucket sync failed [WARN] [RepairJobTask:8] 2023-11-05 13:49:58,132 RepairJob.java:177 - [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6] replication.segments sync failed [WARN] [RepairJobTask:7] 2023-11-05 13:49:58,133 RepairJob.java:177 - [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6] replication.replication_queue sync failed [WARN] [RepairJobTask:11] 2023-11-05 13:49:58,133 RepairJob.java:177 - [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6] replication.payload_register sync failed [ERROR] [Repair#240941:1] 2023-11-05 13:49:58,134 RepairSession.java:321 - [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6] Session completed with the following error org.apache.cassandra.exceptions.RepairException: [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from /<IP>:7000: UNKNOWN at org.apache.cassandra.repair.messages.RepairMessage$1.onFailure(RepairMessage.java:81) at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45) at org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:432) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:522) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:119) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [WARN] [RepairJobTask:3] 2023-11-05 13:49:58,134 RepairJob.java:177 - [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6] replication.replication_queue_ptr sync failed [ERROR] [Repair#240941:1] 2023-11-05 13:49:58,134 RepairRunnable.java:178 - Repair 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 failed: java.lang.RuntimeException: Repair session 7fc76c30-7c14-11ee-a6b1-09b8491c94a6 for range [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]] failed with error [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from /<IP>:7000: UNKNOWN at org.apache.cassandra.repair.RepairRunnable$RepairSessionCallback.onFailure(RepairRunnable.java:698) at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1056) at com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:30) at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1138) at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:958) at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:748) at org.apache.cassandra.repair.RepairSession.forceShutdown(RepairSession.java:342) at org.apache.cassandra.repair.RepairSession$1.onFailure(RepairSession.java:323) at com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1056) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) Caused by: org.apache.cassandra.exceptions.RepairException: [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from /<IP>:7000: UNKNOWN at org.apache.cassandra.repair.messages.RepairMessage$1.onFailure(RepairMessage.java:81) at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97) at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45) at org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:432) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:522) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:119) ... 2 common frames omitted [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,136 Validator.java:237 - Failed creating a merkle tree for [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1cae277e_6e2e_46d2_bffa_983557f6b155, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1cae277e_6e2e_46d2_bffa_983557f6b155, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]], /10.222.45.162:7000 (see log for details) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 ValidationManager.java:173 - Validation failed. java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 CassandraDaemon.java:581 - Exception in thread Thread[ValidationExecutor:66,1,main] java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 Validator.java:237 - Failed creating a merkle tree for [repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1a116929_41ec_4be6_b561_0d42dad8caf7, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/dc_1a116929_41ec_4be6_b561_0d42dad8caf7, [(-892119161849290738,-881535601694419919], (-3078145018521241272,-3075823683795731276], (7303356382144251816,7303733610091843383], (8313290647942866939,8331919689226408479]]], /10.222.45.162:7000 (see log for details) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 ValidationManager.java:173 - Validation failed. java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 CassandraDaemon.java:581 - Exception in thread Thread[ValidationExecutor:66,1,main] java.lang.RuntimeException: Parent repair session with id = 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. at org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) at org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) at org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) at org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) at org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) at org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) at org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) at org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:826) > Merkle Tree repair errors > ------------------------- > > Key: CASSANDRA-19209 > URL: https://issues.apache.org/jira/browse/CASSANDRA-19209 > Project: Cassandra > Issue Type: Bug > Reporter: Kapil Shewate > Priority: Urgent > > On Cassandra 4.0.7 we are seeing Merkle tree repair errors , we are using > cassandra-reaper to do continuous repairs, below is the exception > > ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,131 RepairMessage.java:78 > - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ failed on /<IP>:7000: > UNKNOWN > [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 > RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ > failed on /<IP>:7000: UNKNOWN > [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 > RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ > failed on /<IP>:7000: UNKNOWN > [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 > RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ > failed on /<IP>:7000: UNKNOWN > [ERROR] [RequestResponseStage-9] 2023-11-05 13:49:58,132 > RepairMessage.java:78 - 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 VALIDATION_REQ > failed on /<IP>:7000: UNKNOWN > [WARN] [RepairJobTask:3] 2023-11-05 13:49:58,132 RepairJob.java:177 - repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.payloads_by_bucket sync > failed > [WARN] [RepairJobTask:8] 2023-11-05 13:49:58,132 RepairJob.java:177 - repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.segments sync failed > [WARN] [RepairJobTask:7] 2023-11-05 13:49:58,133 RepairJob.java:177 - repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.replication_queue sync > failed > [WARN] [RepairJobTask:11] 2023-11-05 13:49:58,133 RepairJob.java:177 - repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.payload_register sync failed > [ERROR] [Repair#240941:1] 2023-11-05 13:49:58,134 RepairSession.java:321 - > repair #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 Session completed with the > following error > org.apache.cassandra.exceptions.RepairException: [repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 > on replication/replication_queue, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276], > (7303356382144251816,7303733610091843383], > (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from > /<IP>:7000: UNKNOWN > at > org.apache.cassandra.repair.messages.RepairMessage$1.onFailure(RepairMessage.java:81) > at > org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) > at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78) > at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97) > at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45) > at > org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:432) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:522) > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:119) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.lang.Thread.run(Thread.java:826) > [WARN] [RepairJobTask:3] 2023-11-05 13:49:58,134 RepairJob.java:177 - repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 replication.replication_queue_ptr sync > failed > [ERROR] [Repair#240941:1] 2023-11-05 13:49:58,134 RepairRunnable.java:178 - > Repair 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 failed: > java.lang.RuntimeException: Repair session > 7fc76c30-7c14-11ee-a6b1-09b8491c94a6 for range > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276], > (7303356382144251816,7303733610091843383], > (8313290647942866939,8331919689226408479]] failed with error [repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 > on replication/replication_queue, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276], > (7303356382144251816,7303733610091843383], > (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from > /<IP>:7000: UNKNOWN > at > org.apache.cassandra.repair.RepairRunnable$RepairSessionCallback.onFailure(RepairRunnable.java:698) > at > com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1056) > at > com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:30) > at > com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1138) > at > com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:958) > at > com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:748) > at > org.apache.cassandra.repair.RepairSession.forceShutdown(RepairSession.java:342) > at > org.apache.cassandra.repair.RepairSession$1.onFailure(RepairSession.java:323) > at > com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1056) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.lang.Thread.run(Thread.java:826) > Caused by: org.apache.cassandra.exceptions.RepairException: [repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on replication/replication_queue, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 > on replication/replication_queue, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276], > (7303356382144251816,7303733610091843383], > (8313290647942866939,8331919689226408479]]] Got VALIDATION_REQ failure from > /<IP>:7000: UNKNOWN > at > org.apache.cassandra.repair.messages.RepairMessage$1.onFailure(RepairMessage.java:81) > at > org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) > at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78) > at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97) > at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45) > at > org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:432) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:522) > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:165) > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:137) > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:119) > ... 2 common frames omitted > [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,136 Validator.java:237 - > Failed creating a merkle tree for [repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on > replication/dc_1cae277e_6e2e_46d2_bffa_983557f6b155, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 > on replication/dc_1cae277e_6e2e_46d2_bffa_983557f6b155, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276], > (7303356382144251816,7303733610091843383], > (8313290647942866939,8331919689226408479]]], /<IP>:7000 (see log for details) > [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 > ValidationManager.java:173 - Validation failed. > java.lang.RuntimeException: Parent repair session with id = > 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. > at > org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) > at > org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) > at > org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) > at > org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) > at > org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) > at > org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) > at java.util.concurrent.FutureTask.run(FutureTask.java:277) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.lang.Thread.run(Thread.java:826) > [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 > CassandraDaemon.java:581 - Exception in thread > Thread[ValidationExecutor:66,1,main] > java.lang.RuntimeException: Parent repair session with id = > 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. > at > org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) > at > org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) > at > org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) > at > org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) > at > org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) > at > org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) > at java.util.concurrent.FutureTask.run(FutureTask.java:277) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.lang.Thread.run(Thread.java:826) > [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 Validator.java:237 - > Failed creating a merkle tree for [repair > #7fc76c30-7c14-11ee-a6b1-09b8491c94a6 on > replication/dc_1a116929_41ec_4be6_b561_0d42dad8caf7, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276|#7fc76c30-7c14-11ee-a6b1-09b8491c94a6 > on replication/dc_1a116929_41ec_4be6_b561_0d42dad8caf7, > [(-892119161849290738,-881535601694419919], > (-3078145018521241272,-3075823683795731276], > (7303356382144251816,7303733610091843383], > (8313290647942866939,8331919689226408479]]], /<IP>:7000 (see log for details) > [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 > ValidationManager.java:173 - Validation failed. > java.lang.RuntimeException: Parent repair session with id = > 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. > at > org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) > at > org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) > at > org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) > at > org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) > at > org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) > at > org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) > at java.util.concurrent.FutureTask.run(FutureTask.java:277) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.lang.Thread.run(Thread.java:826) > [ERROR] [ValidationExecutor:66] 2023-11-05 13:49:58,137 > CassandraDaemon.java:581 - Exception in thread > Thread[ValidationExecutor:66,1,main] > java.lang.RuntimeException: Parent repair session with id = > 7fbb5e40-7c14-11ee-a6b1-09b8491c94a6 has failed. > at > org.apache.cassandra.service.ActiveRepairService.getParentRepairSession(ActiveRepairService.java:690) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.getSSTablesToValidate(CassandraValidationIterator.java:116) > at > org.apache.cassandra.db.repair.CassandraValidationIterator.<init>(CassandraValidationIterator.java:203) > at > org.apache.cassandra.db.repair.CassandraTableRepairManager.getValidationIterator(CassandraTableRepairManager.java:51) > at > org.apache.cassandra.repair.ValidationManager.getValidationIterator(ValidationManager.java:89) > at > org.apache.cassandra.repair.ValidationManager.doValidation(ValidationManager.java:112) > at > org.apache.cassandra.repair.ValidationManager.access$000(ValidationManager.java:41) > at > org.apache.cassandra.repair.ValidationManager$1.call(ValidationManager.java:162) > at java.util.concurrent.FutureTask.run(FutureTask.java:277) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1160) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) > at > io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) > at java.lang.Thread.run(Thread.java:826) -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org