[ https://issues.apache.org/jira/browse/CASSANDRA-12192?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15382549#comment-15382549 ]
Philip Thompson commented on CASSANDRA-12192: --------------------------------------------- So, I have the following error in the logs of nodes2 and 3: {code} ERROR [MessagingService-Incoming-/127.0.0.1] 2016-07-13 05:23:15,952 CassandraDaemon.java:201 - Exception in thread Thread[MessagingService-Incoming-/127.0.0.1,5,main] java.lang.RuntimeException: Unknown column cdc during deserialization at org.apache.cassandra.db.Columns$Serializer.deserialize(Columns.java:432) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.SerializationHeader$Serializer.deserializeForMessaging(SerializationHeader.java:427) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.deserializeHeader(UnfilteredRowIteratorSerializer.java:190) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize30(PartitionUpdate.java:657) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize(PartitionUpdate.java:645) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:344) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:353) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:609) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:592) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.MessageIn.read(MessageIn.java:98) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:201) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:178) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:92) ~[apache-cassandra-3.0.8.jar:3.0.8] DEBUG [GossipStage:1] 2016-07-13 05:23:16,855 MigrationManager.java:101 - Submitting migration task for /127.0.0.1 ERROR [MessagingService-Incoming-/127.0.0.1] 2016-07-13 05:23:17,383 CassandraDaemon.java:201 - Exception in thread Thread[MessagingService-Incoming-/127.0.0.1,5,main] java.lang.RuntimeException: Unknown column cdc during deserialization at org.apache.cassandra.db.Columns$Serializer.deserialize(Columns.java:432) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.SerializationHeader$Serializer.deserializeForMessaging(SerializationHeader.java:427) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.deserializeHeader(UnfilteredRowIteratorSerializer.java:190) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize30(PartitionUpdate.java:657) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize(PartitionUpdate.java:645) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:344) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:353) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:609) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:592) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.MessageIn.read(MessageIn.java:98) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:201) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:178) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:92) ~[apache-cassandra-3.0.8.jar:3.0.8] DEBUG [GossipStage:1] 2016-07-13 05:23:17,858 StorageService.java:1899 - Node /127.0.0.1 state NORMAL, token [-1378122049260494817, -1491944349266034000, -1622944791425543487, -2011387665415943034, -2604861794090361113, -2623806159584154975, -2661418387004022901, -3056348922832743393, -3634522926981587558, -3705545868347210901, -3908608253522012139, -4184197016100579239, -4304468738156574476, -5163962544200248432, -5186438155513518780, -7535126099082732347, -8462503993082472101, -846725360334075016, 1052501304817178098, 153582774376769981, 1608837061201738937, 2506254590874653594, 2760740065197146188, 2801316407811925741, 3971348542318844430, 4471389004257102104, 529149305350352167, 5848502616542491401, 7102558182178550651, 8145642075546410367, 8236331454292925125, 999899012538997580] INFO [GossipStage:1] 2016-07-13 05:23:17,858 StorageService.java:1902 - Node /127.0.0.1 state jump to NORMAL DEBUG [PendingRangeCalculator:1] 2016-07-13 05:23:17,865 PendingRangeCalculatorService.java:66 - finished calculation for 4 keyspaces in 0ms DEBUG [GossipStage:1] 2016-07-13 05:23:17,865 MigrationManager.java:101 - Submitting migration task for /127.0.0.1 ERROR [MessagingService-Incoming-/127.0.0.1] 2016-07-13 05:23:17,891 CassandraDaemon.java:201 - Exception in thread Thread[MessagingService-Incoming-/127.0.0.1,5,main] java.lang.RuntimeException: Unknown column cdc during deserialization at org.apache.cassandra.db.Columns$Serializer.deserialize(Columns.java:432) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.SerializationHeader$Serializer.deserializeForMessaging(SerializationHeader.java:427) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.deserializeHeader(UnfilteredRowIteratorSerializer.java:190) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize30(PartitionUpdate.java:657) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize(PartitionUpdate.java:645) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:344) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:353) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:609) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:592) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.MessageIn.read(MessageIn.java:98) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:201) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:178) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:92) ~[apache-cassandra-3.0.8.jar:3.0.8] DEBUG [MemtablePostFlush:1] 2016-07-13 05:23:22,991 ColumnFamilyStore.java:898 - forceFlush requested but everything is clean in test DEBUG [SharedPool-Worker-1] 2016-07-13 05:23:22,993 ColumnFamilyStore.java:2015 - Discarding sstable data for truncated CF + indexes DEBUG [MemtablePostFlush:1] 2016-07-13 05:23:22,994 ColumnFamilyStore.java:898 - forceFlush requested but everything is clean in test DEBUG [SharedPool-Worker-1] 2016-07-13 05:23:23,007 ColumnFamilyStore.java:845 - Enqueuing flush of local: 653 (0%) on-heap, 0 (0%) off-heap DEBUG [MemtableFlushWriter:2] 2016-07-13 05:23:23,007 Memtable.java:353 - Writing Memtable-local@450968517(0.122KiB serialized bytes, 5 ops, 0%/0% of on/off-heap limit) DEBUG [MemtableFlushWriter:2] 2016-07-13 05:23:23,010 Memtable.java:386 - Completed flushing /mnt/tmp/dtest-9TElyd/test/node2/data1/system/local-7ad54392bcdd35a684174e047860b377/mb-14-big-Data.db (0.086KiB) for commitlog position ReplayPosition(segmentId=1468387367914, position=56625) ERROR [MessagingService-Incoming-/127.0.0.1] 2016-07-13 05:23:23,198 CassandraDaemon.java:201 - Exception in thread Thread[MessagingService-Incoming-/127.0.0.1,5,main] java.lang.RuntimeException: Unknown column cdc during deserialization at org.apache.cassandra.db.Columns$Serializer.deserialize(Columns.java:432) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.SerializationHeader$Serializer.deserializeForMessaging(SerializationHeader.java:427) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.rows.UnfilteredRowIteratorSerializer.deserializeHeader(UnfilteredRowIteratorSerializer.java:190) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize30(PartitionUpdate.java:657) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.partitions.PartitionUpdate$PartitionUpdateSerializer.deserialize(PartitionUpdate.java:645) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:344) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.db.Mutation$MutationSerializer.deserialize(Mutation.java:353) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:609) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.service.MigrationManager$MigrationsSerializer.deserialize(MigrationManager.java:592) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.MessageIn.read(MessageIn.java:98) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:201) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.receiveMessages(IncomingTcpConnection.java:178) ~[apache-cassandra-3.0.8.jar:3.0.8] at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:92) ~[apache-cassandra-3.0.8.jar:3.0.8] {code} > dtest failure in > upgrade_tests.cql_tests.TestCQLNodes3RF3_Upgrade_current_3_0_x_To_head_trunk.map_keys_indexing_test > -------------------------------------------------------------------------------------------------------------------- > > Key: CASSANDRA-12192 > URL: https://issues.apache.org/jira/browse/CASSANDRA-12192 > Project: Cassandra > Issue Type: Test > Reporter: Sean McCarthy > Assignee: DS Test Eng > Labels: dtest > Attachments: node1.log, node1_debug.log, node1_gc.log, node2.log, > node2_debug.log, node2_gc.log, node3.log, node3_debug.log, node3_gc.log > > > example failure: > http://cassci.datastax.com/job/upgrade_tests-all/59/testReport/upgrade_tests.cql_tests/TestCQLNodes3RF3_Upgrade_current_3_0_x_To_head_trunk/map_keys_indexing_test > Failed on CassCI build upgrade_tests-all #59 > {code} > Stacktrace > File "/usr/lib/python2.7/unittest/case.py", line 329, in run > testMethod() > File "/home/automaton/cassandra-dtest/tools.py", line 290, in wrapped > f(obj) > File "/home/automaton/cassandra-dtest/upgrade_tests/cql_tests.py", line > 3668, in map_keys_indexing_test > cursor.execute("TRUNCATE test") > File "cassandra/cluster.py", line 1941, in > cassandra.cluster.Session.execute (cassandra/cluster.c:33642) > return self.execute_async(query, parameters, trace, custom_payload, > timeout, execution_profile).result() > File "cassandra/cluster.py", line 3629, in > cassandra.cluster.ResponseFuture.result (cassandra/cluster.c:69369) > raise self._final_exception > '<Error from server: code=1003 [Error during truncate] message="Error during > truncate: Truncate timed out - received only 2 responses"> > {code} > Related failure: > http://cassci.datastax.com/job/upgrade_tests-all/59/testReport/upgrade_tests.cql_tests/TestCQLNodes2RF1_Upgrade_current_3_0_x_To_head_trunk/map_keys_indexing_test/ -- This message was sent by Atlassian JIRA (v6.3.4#6332)