[ https://issues.apache.org/jira/browse/CASSANDRA-8387?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16192225#comment-16192225 ]
Jay Zhuang edited comment on CASSANDRA-8387 at 10/5/17 12:06 AM: ----------------------------------------------------------------- Is there any mitigation other than rolling restart the cluster? We saw the problem in multiple clusters (version 3.0.14), it's causing other table creation returns failure, for example, when create table {{tt81}}, it returns {{tt4}} has problem: {noformat} ./bin/cqlsh 127.0.0.1 -e "create table zjay_test.tt81(tt int PRIMARY KEY, name text, id text)" <stdin>:1:ServerError: java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) {noformat} The table {{tt81}} is successfully created but the schema id is never changed, I guess it's because the migration task failed: {noformat} DEBUG [MigrationStage:1] 2017-10-04 11:44:43,185 Schema.java:465 - Adding org.apache.cassandra.config.CFMetaData@540ff3c8[cfId=1605ec90-a934-11e7-b03f-237cabe3739e,ksName=zjay_test,cfName=tt81,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_ chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, co mpaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [id name]],partitionKeyColumns=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cassandra .db.marshal.Int32Type, kind=PARTITION_KEY, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={},trigg ers=[],indexes=[]] to cfIdMap DEBUG [MigrationStage:1] 2017-10-04 11:44:43,186 CFMetaData.java:745 - applying org.apache.cassandra.config.CFMetaData@6e7e6efb[cfId=8f64bab0-a89b-11e7-bca7-8d4000ea9db6,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_re pair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE' }, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns= [[] | [id name]],partitionKeyColumns=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cass andra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={}, triggers=[],indexes=[]] to org.apache.cassandra.config.CFMetaData@69e66630[cfId=8f5ccb70-a89b-11e7-b03f-237cabe3739e,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_che ck_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassa ndra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [id name]],partitionKeyColumns=[ColumnDefinitio n{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, posit ion=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={},triggers=[],indexes=[]] ERROR [MigrationStage:1] 2017-10-04 11:44:43,187 CassandraDaemon.java:207 - Exception in thread Thread[MigrationStage:1,5,main] org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) at org.apache.cassandra.config.CFMetaData.validateCompatibility(CFMetaData.java:785) ~[main/:na] at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:747) ~[main/:na] at org.apache.cassandra.config.Schema.updateTable(Schema.java:661) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.updateKeyspace(SchemaKeyspace.java:1395) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchema(SchemaKeyspace.java:1351) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchemaAndAnnounceVersion(SchemaKeyspace.java:1301) ~[main/:na] at org.apache.cassandra.service.MigrationManager$1.runMayThrow(MigrationManager.java:543) ~[main/:na] at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) ~[main/:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_141] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_141] at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:79) [main/:na] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_141] ERROR [SharedPool-Worker-7] 2017-10-04 11:44:43,188 QueryMessage.java:128 - Unexpected error during query java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) at org.apache.cassandra.utils.FBUtilities.waitOnFuture(FBUtilities.java:402) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announce(MigrationManager.java:525) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:355) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:340) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:320) ~[main/:na] at org.apache.cassandra.cql3.statements.CreateTableStatement.announceMigration(CreateTableStatement.java:87) ~[main/:na] at org.apache.cassandra.cql3.statements.SchemaAlteringStatement.execute(SchemaAlteringStatement.java:93) ~[main/:na] at org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:206) ~[main/:na] at org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:237) ~[main/:na] at org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:222) ~[main/:na] at org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:115) ~[main/:na] at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:513) [main/:na] at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:407) [main/:na] at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-all-4.0.44.Final.jar:4.0.44.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:357) [netty-all-4.0.44.Final.jar:4.0.44.Final] at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:35) [netty-all-4.0.44.Final.jar:4.0.44.Final] at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:348) [netty-all-4.0.44.Final.jar:4.0.44.Final] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_141] at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164) [main/:na] at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) [main/:na] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_141] {noformat} Schema push also failed on other nodes: {noformat} DEBUG [InternalResponseStage:19] 2017-10-04 14:52:20,324 CFMetaData.java:745 - applying org.apache.cassandra.config.CFMetaData@6c71d136[cfId=8f64bab0-a89b-11e7-bca7-8d4000ea9db6,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal _read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partition Columns=[[] | [id name]],partitionKeyColumns=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apa che.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedCol umns={},triggers=[],indexes=[]] to org.apache.cassandra.config.CFMetaData@24fcccbc[cfId=8f5ccb70-a89b-11e7-b03f-237cabe3739e,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apac he.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [id name]],partitionKeyColumns=[ColumnD efinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KE Y, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={},triggers=[],indexes=[]] ERROR [InternalResponseStage:19] 2017-10-04 14:52:20,325 MigrationTask.java:96 - Configuration exception merging remote schema org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) at org.apache.cassandra.config.CFMetaData.validateCompatibility(CFMetaData.java:785) ~[main/:na] at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:747) ~[main/:na] at org.apache.cassandra.config.Schema.updateTable(Schema.java:661) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.updateKeyspace(SchemaKeyspace.java:1395) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchema(SchemaKeyspace.java:1351) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchemaAndAnnounceVersion(SchemaKeyspace.java:1301) ~[main/:na] at org.apache.cassandra.service.MigrationTask$1.response(MigrationTask.java:92) ~[main/:na] at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) [main/:na] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) [main/:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_141] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_141] at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:79) [main/:na] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_141] {noformat} I tried `nodetool resetlocalschema`, which causes following failure: {noformat} INFO [MigrationStage:1] 2017-10-04 14:50:51,677 ColumnFamilyStore.java:389 - Initializing zjay_test.tt4 ERROR [MigrationStage:1] 2017-10-04 14:50:51,679 CassandraDaemon.java:207 - Exception in thread Thread[MigrationStage:1,5,main] java.lang.RuntimeException: javax.management.InstanceAlreadyExistsException: org.apache.cassandra.db:type=Tables,keyspace=zjay_test,table=tt4 at org.apache.cassandra.db.ColumnFamilyStore.<init>(ColumnFamilyStore.java:439) ~[main/:na] at org.apache.cassandra.db.ColumnFamilyStore.<init>(ColumnFamilyStore.java:361) ~[main/:na] at org.apache.cassandra.db.ColumnFamilyStore.createColumnFamilyStore(ColumnFamilyStore.java:577) ~[main/:na] at org.apache.cassandra.db.ColumnFamilyStore.createColumnFamilyStore(ColumnFamilyStore.java:554) ~[main/:na] at org.apache.cassandra.db.Keyspace.initCf(Keyspace.java:368) ~[main/:na] at org.apache.cassandra.config.Schema.addTable(Schema.java:649) ~[main/:na] at java.util.HashMap$Values.forEach(HashMap.java:980) ~[na:1.8.0_141] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1080) ~[na:1.8.0_141] at org.apache.cassandra.schema.SchemaKeyspace.updateKeyspace(SchemaKeyspace.java:1386) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchema(SchemaKeyspace.java:1351) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchemaAndAnnounceVersion(SchemaKeyspace.java:1301) ~[main/:na] at org.apache.cassandra.service.MigrationManager$1.runMayThrow(MigrationManager.java:543) ~[main/:na] at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) ~[main/:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_141] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_141] at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:79) [main/:na] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_141] Caused by: javax.management.InstanceAlreadyExistsException: org.apache.cassandra.db:type=Tables,keyspace=zjay_test,table=tt4 at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324) ~[na:1.8.0_141] at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522) ~[na:1.8.0_141] at org.apache.cassandra.db.ColumnFamilyStore.<init>(ColumnFamilyStore.java:434) ~[main/:na] ... 18 common frames omitted {noformat} (maybe CASSANDRA-13834 is the fix) and inconsistent schema ID: {noformat} Cluster Information: Name: tt2 Snitch: org.apache.cassandra.locator.DynamicEndpointSnitch Partitioner: org.apache.cassandra.dht.Murmur3Partitioner Schema versions: 9a47fb8d-1712-362e-b422-6331fe448d10: [127.0.0.3] 776aa76b-1c7a-3bf8-9891-4564b72981eb: [127.0.0.1] 59adb24e-f3cd-3e02-97f0-5b395827453f: [127.0.0.2] {noformat} Here is the reproduce steps with local ccm nodes (3 nodes): {noformat} CREATE KEYSPACE zjay_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} echo 1,2,3, | parallel -vkd, "/Users/zjay/ws/cassandra/bin/cqlsh 127.0.0.{} -e 'create table zjay_test.tt4(tt int PRIMARY KEY, name text, id text)'" {noformat} Rolling restart the cluster will fix the problem, just wondering if there's any better way to recover from this issue. was (Author: jay.zhuang): Is there any mitigation other than rolling restart the cluster? We saw the problem in multiple clusters (version 3.0.14), it's causing other table creation returns failure, for example, when create table {{tt81}}, it returns {{tt4}} has problem: {noformat} ./bin/cqlsh 127.0.0.1 -e "create table zjay_test.tt81(tt int PRIMARY KEY, name text, id text)" <stdin>:1:ServerError: java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) {noformat} The table {{tt81}} is successfully created but the schema id is never changed, I guess it's because the migration task failed: {noformat} DEBUG [MigrationStage:1] 2017-10-04 11:44:43,185 Schema.java:465 - Adding org.apache.cassandra.config.CFMetaData@540ff3c8[cfId=1605ec90-a934-11e7-b03f-237cabe3739e,ksName=zjay_test,cfName=tt81,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_ chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, co mpaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [id name]],partitionKeyColumns=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cassandra .db.marshal.Int32Type, kind=PARTITION_KEY, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={},trigg ers=[],indexes=[]] to cfIdMap DEBUG [MigrationStage:1] 2017-10-04 11:44:43,186 CFMetaData.java:745 - applying org.apache.cassandra.config.CFMetaData@6e7e6efb[cfId=8f64bab0-a89b-11e7-bca7-8d4000ea9db6,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_re pair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE' }, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns= [[] | [id name]],partitionKeyColumns=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cass andra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={}, triggers=[],indexes=[]] to org.apache.cassandra.config.CFMetaData@69e66630[cfId=8f5ccb70-a89b-11e7-b03f-237cabe3739e,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_che ck_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassa ndra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [id name]],partitionKeyColumns=[ColumnDefinitio n{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, posit ion=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={},triggers=[],indexes=[]] ERROR [MigrationStage:1] 2017-10-04 11:44:43,187 CassandraDaemon.java:207 - Exception in thread Thread[MigrationStage:1,5,main] org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) at org.apache.cassandra.config.CFMetaData.validateCompatibility(CFMetaData.java:785) ~[main/:na] at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:747) ~[main/:na] at org.apache.cassandra.config.Schema.updateTable(Schema.java:661) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.updateKeyspace(SchemaKeyspace.java:1395) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchema(SchemaKeyspace.java:1351) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchemaAndAnnounceVersion(SchemaKeyspace.java:1301) ~[main/:na] at org.apache.cassandra.service.MigrationManager$1.runMayThrow(MigrationManager.java:543) ~[main/:na] at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) ~[main/:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_141] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_141] at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:79) [main/:na] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_141] ERROR [SharedPool-Worker-7] 2017-10-04 11:44:43,188 QueryMessage.java:128 - Unexpected error during query java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) at org.apache.cassandra.utils.FBUtilities.waitOnFuture(FBUtilities.java:402) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announce(MigrationManager.java:525) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:355) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:340) ~[main/:na] at org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:320) ~[main/:na] at org.apache.cassandra.cql3.statements.CreateTableStatement.announceMigration(CreateTableStatement.java:87) ~[main/:na] at org.apache.cassandra.cql3.statements.SchemaAlteringStatement.execute(SchemaAlteringStatement.java:93) ~[main/:na] at org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:206) ~[main/:na] at org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:237) ~[main/:na] at org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:222) ~[main/:na] at org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:115) ~[main/:na] at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:513) [main/:na] at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:407) [main/:na] at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-all-4.0.44.Final.jar:4.0.44.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:357) [netty-all-4.0.44.Final.jar:4.0.44.Final] at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:35) [netty-all-4.0.44.Final.jar:4.0.44.Final] at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:348) [netty-all-4.0.44.Final.jar:4.0.44.Final] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_141] at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164) [main/:na] at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) [main/:na] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_141] {noformat} Schema push also failed on other nodes: {noformat} DEBUG [InternalResponseStage:19] 2017-10-04 14:52:20,324 CFMetaData.java:745 - applying org.apache.cassandra.config.CFMetaData@6c71d136[cfId=8f64bab0-a89b-11e7-bca7-8d4000ea9db6,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal _read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partition Columns=[[] | [id name]],partitionKeyColumns=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apa che.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedCol umns={},triggers=[],indexes=[]] to org.apache.cassandra.config.CFMetaData@24fcccbc[cfId=8f5ccb70-a89b-11e7-b03f-237cabe3739e,ksName=zjay_test,cfName=tt4,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apac he.cassandra.db.compaction.SizeTieredCompactionStrategy, options={max_threshold=32, min_threshold=4}}, compression=org.apache.cassandra.schema.CompressionParams@78dfc45f, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [id name]],partitionKeyColumns=[ColumnD efinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KEY, position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.Int32Type,columnMetadata=[ColumnDefinition{name=tt, type=org.apache.cassandra.db.marshal.Int32Type, kind=PARTITION_KE Y, position=0}, ColumnDefinition{name=id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, ColumnDefinition{name=name, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}],droppedColumns={},triggers=[],indexes=[]] ERROR [InternalResponseStage:19] 2017-10-04 14:52:20,325 MigrationTask.java:96 - Configuration exception merging remote schema org.apache.cassandra.exceptions.ConfigurationException: Column family ID mismatch for keyspace zjay_test table tt4 (found 8f64bab0-a89b-11e7-bca7-8d4000ea9db6; expected 8f5ccb70-a89b-11e7-b03f-237cabe3739e) at org.apache.cassandra.config.CFMetaData.validateCompatibility(CFMetaData.java:785) ~[main/:na] at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:747) ~[main/:na] at org.apache.cassandra.config.Schema.updateTable(Schema.java:661) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.updateKeyspace(SchemaKeyspace.java:1395) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchema(SchemaKeyspace.java:1351) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchemaAndAnnounceVersion(SchemaKeyspace.java:1301) ~[main/:na] at org.apache.cassandra.service.MigrationTask$1.response(MigrationTask.java:92) ~[main/:na] at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:53) [main/:na] at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:67) [main/:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_141] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_141] at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:79) [main/:na] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_141] {noformat} I tried `nodetool resetlocalschema`, which causes following failure: {noformat} INFO [MigrationStage:1] 2017-10-04 14:50:51,677 ColumnFamilyStore.java:389 - Initializing zjay_test.tt4 ERROR [MigrationStage:1] 2017-10-04 14:50:51,679 CassandraDaemon.java:207 - Exception in thread Thread[MigrationStage:1,5,main] java.lang.RuntimeException: javax.management.InstanceAlreadyExistsException: org.apache.cassandra.db:type=Tables,keyspace=zjay_test,table=tt4 at org.apache.cassandra.db.ColumnFamilyStore.<init>(ColumnFamilyStore.java:439) ~[main/:na] at org.apache.cassandra.db.ColumnFamilyStore.<init>(ColumnFamilyStore.java:361) ~[main/:na] at org.apache.cassandra.db.ColumnFamilyStore.createColumnFamilyStore(ColumnFamilyStore.java:577) ~[main/:na] at org.apache.cassandra.db.ColumnFamilyStore.createColumnFamilyStore(ColumnFamilyStore.java:554) ~[main/:na] at org.apache.cassandra.db.Keyspace.initCf(Keyspace.java:368) ~[main/:na] at org.apache.cassandra.config.Schema.addTable(Schema.java:649) ~[main/:na] at java.util.HashMap$Values.forEach(HashMap.java:980) ~[na:1.8.0_141] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1080) ~[na:1.8.0_141] at org.apache.cassandra.schema.SchemaKeyspace.updateKeyspace(SchemaKeyspace.java:1386) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchema(SchemaKeyspace.java:1351) ~[main/:na] at org.apache.cassandra.schema.SchemaKeyspace.mergeSchemaAndAnnounceVersion(SchemaKeyspace.java:1301) ~[main/:na] at org.apache.cassandra.service.MigrationManager$1.runMayThrow(MigrationManager.java:543) ~[main/:na] at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) ~[main/:na] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_141] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_141] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_141] at org.apache.cassandra.concurrent.NamedThreadFactory.lambda$threadLocalDeallocator$0(NamedThreadFactory.java:79) [main/:na] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_141] Caused by: javax.management.InstanceAlreadyExistsException: org.apache.cassandra.db:type=Tables,keyspace=zjay_test,table=tt4 at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900) ~[na:1.8.0_141] at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324) ~[na:1.8.0_141] at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522) ~[na:1.8.0_141] at org.apache.cassandra.db.ColumnFamilyStore.<init>(ColumnFamilyStore.java:434) ~[main/:na] ... 18 common frames omitted {noformat} and inconsistent schema ID: {noformat} Cluster Information: Name: tt2 Snitch: org.apache.cassandra.locator.DynamicEndpointSnitch Partitioner: org.apache.cassandra.dht.Murmur3Partitioner Schema versions: 9a47fb8d-1712-362e-b422-6331fe448d10: [127.0.0.3] 776aa76b-1c7a-3bf8-9891-4564b72981eb: [127.0.0.1] 59adb24e-f3cd-3e02-97f0-5b395827453f: [127.0.0.2] {noformat} Here is the reproduce steps with local ccm nodes (3 nodes): {noformat} CREATE KEYSPACE zjay_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} echo 1,2,3, | parallel -vkd, "/Users/zjay/ws/cassandra/bin/cqlsh 127.0.0.{} -e 'create table zjay_test.tt4(tt int PRIMARY KEY, name text, id text)'" {noformat} Rolling restart the cluster will fix the problem, just wondering if there's any better way to recover from this issue. > Schema inconsistency (cached vs schema_columnfamilies) > ------------------------------------------------------ > > Key: CASSANDRA-8387 > URL: https://issues.apache.org/jira/browse/CASSANDRA-8387 > Project: Cassandra > Issue Type: Bug > Environment: C* 2.1.1 3-node cluster > Reporter: Marcus Olsson > > While running some tests on a 3-node cluster running C* 2.1.1 we encountered > a problem creating the same table schema twice(on different nodes). One thing > to note is that one of the nodes clock was ~4 seconds behind the others, but > I don't think that's the problem since the exception was reproduced here > aswell: http://www.mail-archive.com/user@cassandra.apache.org/msg39560.html. > While running the same create table statement more than once(on different > clients) the logs outputted this on one of the nodes: > {noformat} > (node x.x.x.1): > 2014-11-25T16:11:44.651+0100 INFO [SharedPool-Worker-2] > MigrationManager.java:248 Create new ColumnFamily: > org.apache.cassandra.config.CFMetaData@45c290de[cfId=5e334b40-74b5-11e4-b1b6-017ad0689f5d,ksName=test,cfName=test,cfType=Standard,comparator=org.apache.cassandra.db.marshal.UTF8Type,comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=id, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=CLUSTERING_COLUMN, > componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type, > kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=value, type=org.apache.cassandra.db.marshal.BytesType, > kind=COMPACT_VALUE, componentIndex=null, indexName=null, > indexType=null}],compactionStrategyClass=class > org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.01,memtableFlushPeriod=0,caching={"keys":"ALL", > > "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=true] > ... > 2014-11-25T16:11:44.667+0100 INFO [MigrationStage:1] DefsTables.java:373 > Loading > org.apache.cassandra.config.CFMetaData@40a1ee90[cfId=5bc7c980-74b5-11e4-9131-d9b94a3d8927,ksName=test,cfName=test,cfType=Standard,comparator=org.apache.cassandra.db.marshal.UTF8Type,comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=id, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=CLUSTERING_COLUMN, > componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type, > kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=value, type=org.apache.cassandra.db.marshal.BytesType, > kind=COMPACT_VALUE, componentIndex=null, indexName=null, > indexType=null}],compactionStrategyClass=class > org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.01,memtableFlushPeriod=0,caching={"keys":"ALL", > > "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=true] > ... > java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException: Column family ID > mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1171) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.updateColumnFamily(DefsTables.java:422) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeColumnFamilies(DefsTables.java:295) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchemaInternal(DefsTables.java:194) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchema(DefsTables.java:166) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.service.MigrationManager$2.runMayThrow(MigrationManager.java:393) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > ~[na:1.7.0_51] > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > [na:1.7.0_51] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > Caused by: org.apache.cassandra.exceptions.ConfigurationException: Column > family ID mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.validateCompatility(CFMetaData.java:1254) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:1186) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1167) > ~[apache-cassandra-2.1.1.jar:2.1.1] > ... 11 common frames omitted > 2014-11-25T16:11:44.859+0100 ERROR [SharedPool-Worker-2] > ErrorMessage.java:243 Unexpected exception during request > java.lang.RuntimeException: java.util.concurrent.ExecutionException: > java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException: Column family ID > mismatch (found 5e334b40-74b5-11e4- > b1b6-017ad0689f5d; expected 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.utils.FBUtilities.waitOnFuture(FBUtilities.java:397) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.service.MigrationManager.announce(MigrationManager.java:374) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.service.MigrationManager.announceNewColumnFamily(MigrationManager.java:249) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.cql3.statements.CreateTableStatement.announceMigration(CreateTableStatement.java:114) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.cql3.statements.SchemaAlteringStatement.execute(SchemaAlteringStatement.java:80) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:226) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:481) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.transport.messages.ExecuteMessage.execute(ExecuteMessage.java:133) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:439) > [apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:335) > [apache-cassandra-2.1.1.jar:2.1.1] > at > io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > [na:1.7.0_51] > at > org.apache.cassandra.concurrent.AbstractTracingAwareExecutorService$FutureTask.run(AbstractTracingAwareExecutorService.java:164) > [apache-cassandra-2.1.1.jar:2.1.1] > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) > [apache-cassandra-2.1.1.jar:2.1.1] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > Caused by: java.util.concurrent.ExecutionException: > java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException: Column family ID > mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d > ; expected 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at java.util.concurrent.FutureTask.report(FutureTask.java:122) > ~[na:1.7.0_51] > at java.util.concurrent.FutureTask.get(FutureTask.java:188) > ~[na:1.7.0_51] > at > org.apache.cassandra.utils.FBUtilities.waitOnFuture(FBUtilities.java:393) > ~[apache-cassandra-2.1.1.jar:2.1.1] > ... 17 common frames omitted > Caused by: java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException: Column family ID > mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1171) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.updateColumnFamily(DefsTables.java:422) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeColumnFamilies(DefsTables.java:295) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchemaInternal(DefsTables.java:194) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchema(DefsTables.java:166) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.service.MigrationManager$2.runMayThrow(MigrationManager.java:393) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > [na:1.7.0_51] > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > ~[na:1.7.0_51] > ... 1 common frames omitted > Caused by: org.apache.cassandra.exceptions.ConfigurationException: Column > family ID mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.validateCompatility(CFMetaData.java:1254) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:1186) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1167) > ~[apache-cassandra-2.1.1.jar:2.1.1] > ... 11 common frames omitted > {noformat} > In my understanding this means that it first created the table(with UUID > 5e..) and then got a schema update from another node with UUID 5b.. . > And the node who created the 5b.. table had this: > {noformat} > (node x.x.x.2): > 2014-11-25T16:11:40.580+0100 INFO [SharedPool-Worker-1] > MigrationManager.java:248 Create new ColumnFamily: > org.apache.cassandra.config.CFMetaData@225c1975[cfId=5bc7c980-74b5-11e4-9131-d9b94a3d8927,ksName=test,cfName=test,cfType=Standard,comparator=org.apache.cassandra.db.marshal.UTF8Type,comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=id, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=CLUSTERING_COLUMN, > componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type, > kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=value, type=org.apache.cassandra.db.marshal.BytesType, > kind=COMPACT_VALUE, componentIndex=null, indexName=null, > indexType=null}],compactionStrategyClass=class > org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.01,memtableFlushPeriod=0,caching={"keys":"ALL", > > "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=true] > ... > java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException: Column family ID > mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1171) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.updateColumnFamily(DefsTables.java:422) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeColumnFamilies(DefsTables.java:295) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchemaInternal(DefsTables.java:194) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchema(DefsTables.java:166) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefinitionsUpdateVerbHandler$1.runMayThrow(DefinitionsUpdateVerbHandler.java:49) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > ~[na:1.7.0_51] > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > [na:1.7.0_51] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > Caused by: org.apache.cassandra.exceptions.ConfigurationException: Column > family ID mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.validateCompatility(CFMetaData.java:1254) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:1186) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1167) > ~[apache-cassandra-2.1.1.jar:2.1.1] > ... 11 common frames omitted > {noformat} > And the last node had this: > {noformat} > (node x.x.x.3): > 2014-11-25T16:11:44.363+0100 INFO [MigrationStage:1] DefsTables.java:373 > Loading > org.apache.cassandra.config.CFMetaData@767512bf[cfId=5bc7c980-74b5-11e4-9131-d9b94a3d8927,ksName=test,cfName=test,cfType=Standard,comparator=org.apache.cassandra.db.marshal.UTF8Type,comment=,readRepairChance=0.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=id, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=CLUSTERING_COLUMN, > componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=key, type=org.apache.cassandra.db.marshal.UTF8Type, > kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}, > ColumnDefinition{name=value, type=org.apache.cassandra.db.marshal.BytesType, > kind=COMPACT_VALUE, componentIndex=null, indexName=null, > indexType=null}],compactionStrategyClass=class > org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.01,memtableFlushPeriod=0,caching={"keys":"ALL", > > "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=99.0PERCENTILE,droppedColumns={},triggers=[],isDense=true] > ... > java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException: Column family ID > mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1171) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.updateColumnFamily(DefsTables.java:422) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeColumnFamilies(DefsTables.java:295) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchemaInternal(DefsTables.java:194) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefsTables.mergeSchema(DefsTables.java:166) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.db.DefinitionsUpdateVerbHandler$1.runMayThrow(DefinitionsUpdateVerbHandler.java:49) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) > ~[na:1.7.0_51] > at java.util.concurrent.FutureTask.run(FutureTask.java:262) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > ~[na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > [na:1.7.0_51] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > Caused by: org.apache.cassandra.exceptions.ConfigurationException: Column > family ID mismatch (found 5e334b40-74b5-11e4-b1b6-017ad0689f5d; expected > 5bc7c980-74b5-11e4-9131-d9b94a3d8927) > at > org.apache.cassandra.config.CFMetaData.validateCompatility(CFMetaData.java:1254) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at org.apache.cassandra.config.CFMetaData.apply(CFMetaData.java:1186) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.config.CFMetaData.reload(CFMetaData.java:1167) > ~[apache-cassandra-2.1.1.jar:2.1.1] > ... 11 common frames omitted > {noformat} > Not a problem at all, since they all seemed to agree that 5b.. was the schema > to go with and everything went fine with inserting data into the table. The > data directory contained the 5b.. directory and not 5e.. (on all nodes). The > problem occured later when node x.x.x.3 was restarted it created the > directory test/test-5e.. in the data directory. When performing a repair on > the node it gave these kinds of exceptions: > {noformat} > (node x.x.x.3): > 2014-11-25T16:21:31.726+0100 WARN [STREAM-IN-/x.x.x.2] > StreamSession.java:590 [Stream #bc054793-74b6-11e4-baf6-fbb25862655d] > Retrying for following error > java.io.IOException: CF 5bc7c980-74b5-11e4-9131-d9b94a3d8927 was dropped > during streaming > at > org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:71) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:48) > [apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:38) > [apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55) > [apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:245) > [apache-cassandra-2.1.1.jar:2.1.1] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > ... > 2014-11-25T16:21:31.726+0100 ERROR [STREAM-IN-/x.x.x.2] > StreamSession.java:470 [Stream #bc054793-74b6-11e4-baf6-fbb25862655d] > Streaming error occurred > java.lang.IllegalArgumentException: Unknown type 0 > at > org.apache.cassandra.streaming.messages.StreamMessage$Type.get(StreamMessage.java:89) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:54) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:245) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > ... > 2014-11-25T16:21:31.738+0100 ERROR [AntiEntropySessions:1] > RepairSession.java:303 [repair #bbf34630-74b6-11e4-baf6-fbb25862655d] session > completed with the following error > org.apache.cassandra.exceptions.RepairException: [repair > #bbf34630-74b6-11e4-baf6-fbb25862655d on test/test, > (3708140006327119811,3721360916800446611]] Sync failed between /x.x.x.3 and > /x.x.x.2 > at > org.apache.cassandra.repair.RepairSession.syncComplete(RepairSession.java:223) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:389) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:126) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:62) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) > [na:1.7.0_51] > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) > [na:1.7.0_51] > at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51] > {noformat} > And the other nodes had exceptions like: > {noformat} > (node x.x.x.2) > 2014-11-25T16:21:28.145+0100 ERROR [STREAM-OUT-/x.x.x.3] > StreamSession.java:470 [Stream #bc309d50-74b6-11e4-baf6-fbb25862655d] > Streaming error occurred > java.io.IOException: Connection reset by peer > at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[na:1.7.0_51] > at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) > ~[na:1.7.0_51] > at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) > ~[na:1.7.0_51] > at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[na:1.7.0_51] > at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:487) > ~[na:1.7.0_51] > at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) > ~[na:1.7.0_51] > at java.nio.channels.Channels.writeFully(Channels.java:98) > ~[na:1.7.0_51] > at java.nio.channels.Channels.access$000(Channels.java:61) > ~[na:1.7.0_51] > at java.nio.channels.Channels$1.write(Channels.java:174) > ~[na:1.7.0_51] > at java.io.OutputStream.write(OutputStream.java:75) ~[na:1.7.0_51] > at java.nio.channels.Channels$1.write(Channels.java:155) > ~[na:1.7.0_51] > at > org.apache.cassandra.io.util.DataOutputStreamPlus.write(DataOutputStreamPlus.java:45) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.io.util.AbstractDataOutput.writeLong(AbstractDataOutput.java:232) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.io.compress.CompressionMetadata$ChunkSerializer.serialize(CompressionMetadata.java:406) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.io.compress.CompressionMetadata$ChunkSerializer.serialize(CompressionMetadata.java:402) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.compress.CompressionInfo$CompressionInfoSerializer.serialize(CompressionInfo.java:60) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.compress.CompressionInfo$CompressionInfoSerializer.serialize(CompressionInfo.java:47) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.messages.FileMessageHeader$FileMessageHeaderSerializer.serialize(FileMessageHeader.java:135) > ~[apache-cassandra-2.1.1.jar:2.1.1] > at > org.apache.cassandra.streaming.messages.FileMessageHeader$FileMessageHeaderSerializer.serialize(FileMessageHeader.java:120) > ~[apache-cassandra-2.1.1.jar:2.1.1] > {noformat} > In System.schema_columnfamilies the id for test.test is 5e.. on all nodes, > but the nodes that hadn't restarted still used the 5b.. id for the table > (atleast in the data directory, probably a cached id). > The problem seems to be that the schema change for 5e.. is rejected on all > nodes (java.lang.RuntimeException: > org.apache.cassandra.exceptions.ConfigurationException thrown), while the > mutation for it still gets applied in schema_columnfamilies. -- This message was sent by Atlassian JIRA (v6.4.14#64029) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org