[ https://issues.apache.org/jira/browse/CASSANDRA-15594?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17044780#comment-17044780 ]
David Capwell commented on CASSANDRA-15594: ------------------------------------------- I was able to replicate this with the following test {code} @Test public void userTest() throws Throwable { try (Cluster cluster = init(Cluster.build(3).withConfig(c -> c.set("memtable_allocation_type", "offheap_objects")).start())) { // memtable_allocation_type = offheap_objects cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck1 text, ck2 text, v int, PRIMARY KEY (pk, ck1, ck2))"); String value = "This is some text which keeps building up. "; for (int i = 0; i < 1000; i++) { value += "This is some text which keeps building up. "; } cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk, ck1, ck2) VALUES (?, ?, ?)", ConsistencyLevel.LOCAL_QUORUM, 42, value, value); } } {code} > Uncaught exception on thread Thread[MutationStage-1,5,main] > ----------------------------------------------------------- > > Key: CASSANDRA-15594 > URL: https://issues.apache.org/jira/browse/CASSANDRA-15594 > Project: Cassandra > Issue Type: Bug > Components: Legacy/Local Write-Read Paths, Local/Memtable > Reporter: Aleksandr > Priority: Normal > Labels: 3.11.4 > > Hi. > We have a cluster of 3 machines running Cassandra 3.11.4 for several months, > cpp-driver 2.12.0 is used to communicate with Cassandra. > We have 13 tables in total. > Recently we've noticed that one particular INSERT operation fails with the > error "Write failure: Operation failed - received 0 responses and 2 failures" > - we retry twice the same query with timeouts 1 and 5.1 seconds. The query is: > {code:java} > INSERT INTO keyspace1.table1 (column1, column2, column3, column4, column5) > VALUES (?, ?, ?, ?, ?) IF NOT EXISTS{code} > Every time this particular INSERT happens we see the error in the log: > {code:java} > WARN [MutationStage-1] 2020-02-21 12:13:46,435 > AbstractLocalAwareExecutorService.java:167 - Uncaught exception on thread > Thread[MutationStage-1,5,main]: {} > java.lang.AssertionError: null > at org.apache.cassandra.db.NativeClustering.<init>(NativeClustering.java:45) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.utils.memory.NativeAllocator$CloningBTreeRowBuilder.newRow(NativeAllocator.java:80) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.rows.Rows.copy(Rows.java:43) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition$RowUpdater.apply(AtomicBTreePartition.java:332) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition$RowUpdater.apply(AtomicBTreePartition.java:295) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.utils.btree.NodeBuilder.addNewKey(NodeBuilder.java:323) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.NodeBuilder.update(NodeBuilder.java:184) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.TreeBuilder.update(TreeBuilder.java:95) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.BTree.update(BTree.java:182) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition.addAllWithSizeDelta(AtomicBTreePartition.java:156) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Memtable.put(Memtable.java:282) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1352) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.applyInternal(Keyspace.java:626) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:470) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:450) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.service.paxos.PaxosState.commit(PaxosState.java:150) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.service.StorageProxy$6.runMayThrow(StorageProxy.java:571) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.service.StorageProxy$LocalMutationRunnable.run(StorageProxy.java:2637) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[na:1.8.0_232] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:162) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:134) > [apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:114) > [apache-cassandra-3.11.4.jar:3.11.4] > at java.lang.Thread.run(Thread.java:748) [na:1.8.0_232] > WARN [MutationStage-4] 2020-02-21 12:13:46,449 > AbstractLocalAwareExecutorService.java:167 - Uncaught exception on thread > Thread[MutationStage-4,5,main]: {} > java.lang.AssertionError: null > at org.apache.cassandra.db.NativeClustering.<init>(NativeClustering.java:45) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.utils.memory.NativeAllocator$CloningBTreeRowBuilder.newRow(NativeAllocator.java:80) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.rows.Rows.copy(Rows.java:43) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition$RowUpdater.apply(AtomicBTreePartition.java:332) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition$RowUpdater.apply(AtomicBTreePartition.java:295) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.utils.btree.NodeBuilder.addNewKey(NodeBuilder.java:323) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.NodeBuilder.update(NodeBuilder.java:184) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.TreeBuilder.update(TreeBuilder.java:95) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.BTree.update(BTree.java:182) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition.addAllWithSizeDelta(AtomicBTreePartition.java:156) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Memtable.put(Memtable.java:282) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1352) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.applyInternal(Keyspace.java:626) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:470) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:450) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.service.paxos.PaxosState.commit(PaxosState.java:150) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.service.paxos.CommitVerbHandler.doVerb(CommitVerbHandler.java:33) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:66) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[na:1.8.0_232] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:162) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:134) > [apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:114) > [apache-cassandra-3.11.4.jar:3.11.4] > at java.lang.Thread.run(Thread.java:748) [na:1.8.0_232] > WARN [MutationStage-5] 2020-02-21 12:13:46,461 > AbstractLocalAwareExecutorService.java:167 - Uncaught exception on thread > Thread[MutationStage-5,5,main]: {} > java.lang.AssertionError: null > at org.apache.cassandra.db.NativeClustering.<init>(NativeClustering.java:45) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.utils.memory.NativeAllocator$CloningBTreeRowBuilder.newRow(NativeAllocator.java:80) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.rows.Rows.copy(Rows.java:43) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition$RowUpdater.apply(AtomicBTreePartition.java:332) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition$RowUpdater.apply(AtomicBTreePartition.java:295) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.utils.btree.NodeBuilder.addNewKey(NodeBuilder.java:323) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.NodeBuilder.update(NodeBuilder.java:184) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.TreeBuilder.update(TreeBuilder.java:95) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.utils.btree.BTree.update(BTree.java:182) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.partitions.AtomicBTreePartition.addAllWithSizeDelta(AtomicBTreePartition.java:156) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Memtable.put(Memtable.java:282) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:1352) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.applyInternal(Keyspace.java:626) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:470) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.db.Keyspace.apply(Keyspace.java:450) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.service.paxos.PaxosState.commit(PaxosState.java:150) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.service.paxos.CommitVerbHandler.doVerb(CommitVerbHandler.java:33) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:66) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > ~[na:1.8.0_232] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:162) > ~[apache-cassandra-3.11.4.jar:3.11.4] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$LocalSessionFutureTask.run(AbstractLocalAwareExecutorService.java:134) > [apache-cassandra-3.11.4.jar:3.11.4] > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:114) > [apache-cassandra-3.11.4.jar:3.11.4] > at java.lang.Thread.run(Thread.java:748) [na:1.8.0_232]{code} > > The following table schema is used for this table: > {code:java} > CREATE TABLE keyspace1.table1 ( > column1 text, > column2 bigint, > column3 text, > column4 uuid, > column5 bigint, > PRIMARY KEY (column1, column2, column3) > ) WITH CLUSTERING ORDER BY (column2 ASC, column3 ASC) > AND bloom_filter_fp_chance = 0.01 > AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} > AND comment = '' > AND compaction = {'class': > 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', > 'max_threshold': '32', 'min_threshold': '4'} > AND compression = {'chunk_length_in_kb': '64', 'class': > 'org.apache.cassandra.io.compress.LZ4Compressor'} > AND crc_check_chance = 1.0 > AND dclocal_read_repair_chance = 0.1 > AND default_time_to_live = 0 > AND gc_grace_seconds = 86400 > AND max_index_interval = 2048 > AND memtable_flush_period_in_ms = 0 > AND min_index_interval = 128 > AND read_repair_chance = 0.0 > AND speculative_retry = '99PERCENTILE';{code} > > Two small additions: > 1) We couldn't reproduce this issue on a different cluster with 3 machines - > everything works fine. No issues when only machine is used either. > 2) Only this one INSERT fails for this table. There are others similar > INSERTS into the same table, but they don't fail. -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org