[ https://issues.apache.org/jira/browse/CASSANDRA-12083?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Sharvanath Pathak updated CASSANDRA-12083: ------------------------------------------ Component/s: Core > Race condition during system.roles column family creation > --------------------------------------------------------- > > Key: CASSANDRA-12083 > URL: https://issues.apache.org/jira/browse/CASSANDRA-12083 > Project: Cassandra > Issue Type: Bug > Components: Core > Reporter: Sharvanath Pathak > Fix For: 3.0.6 > > > There is an issue where Cassandra fails with the following exception on > startup: > {noformat} > DEBUG [InternalResponseStage:2] 2016-06-20 09:43:00,651 Schema.java:465 - > Adding > org.apache.cassandra.config.CFMetaData@2882b66d[cfId=5bc52802-de25-35ed-aeab-188eecebb090,ksName=system_auth,cfName=roles,flags=[COMPOUND],params=TableParams{comment=role > definitions, read_repair_chance=0.0, dclocal_read_repair_chance=0.0, > bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=7776000, > default_time_to_live=0, memtable_flush_period_in_ms=3600000, > min_index_interval=128, max_index_interval=2048, > speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' > : 'NONE'}, > compaction=CompactionParams{class=org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, > options={max_threshold=32, min_threshold=4}}, > compression=org.apache.cassandra.schema.CompressionParams@e6e0212, > extensions={}},comparator=comparator(),partitionColumns=[[] | [can_login > is_superuser salted_hash > member_of]],partitionKeyColumns=[ColumnDefinition{name=role, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=PARTITION_KEY, > position=0}],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.UTF8Type,columnMetadata=[ColumnDefinition{name=role, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=PARTITION_KEY, > position=0}, ColumnDefinition{name=salted_hash, > type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, position=-1}, > ColumnDefinition{name=member_of, > type=org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.UTF8Type), > kind=REGULAR, position=-1}, ColumnDefinition{name=can_login, > type=org.apache.cassandra.db.marshal.BooleanType, kind=REGULAR, position=-1}, > ColumnDefinition{name=is_superuser, > type=org.apache.cassandra.db.marshal.BooleanType, kind=REGULAR, > position=-1}],droppedColumns={},triggers=[],indexes=[]] to cfIdMap > INFO [InternalResponseStage:2] 2016-06-20 09:43:00,653 > ColumnFamilyStore.java:381 - Initializing system_auth.roles > DEBUG [InternalResponseStage:2] 2016-06-20 09:43:00,664 > MigrationManager.java:556 - Gossiping my schema version > c2a2bb4f-7d31-3fb8-a216-00b41a643650 > DEBUG [InternalResponseStage:1] 2016-06-20 09:43:00,669 > ColumnFamilyStore.java:831 - Enqueuing flush of keyspaces: 1566 (0%) on-heap, > 0 (0%) off-heap > DEBUG [MemtableFlushWriter:2] 2016-06-20 09:43:00,669 Memtable.java:372 - > Writing Memtable-keyspaces@650010305(0.437KiB serialized bytes, 3 ops, 0%/0% > of on/off-heap limit) > ERROR [main] 2016-06-20 09:43:00,670 CassandraDaemon.java:692 - Exception > encountered during startup > java.lang.IllegalArgumentException: Unknown CF > 5bc52802-de25-35ed-aeab-188eecebb090 > at > org.apache.cassandra.db.Keyspace.getColumnFamilyStore(Keyspace.java:206) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.db.Keyspace.getColumnFamilyStore(Keyspace.java:199) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.cql3.restrictions.StatementRestrictions.<init>(StatementRestrictions.java:168) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.cql3.statements.SelectStatement$RawStatement.prepareRestrictions(SelectStatement.java:874) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.cql3.statements.SelectStatement$RawStatement.prepare(SelectStatement.java:821) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.cql3.statements.SelectStatement$RawStatement.prepare(SelectStatement.java:809) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.auth.CassandraRoleManager.prepare(CassandraRoleManager.java:446) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.auth.CassandraRoleManager.setup(CassandraRoleManager.java:144) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.StorageService.doAuthSetup(StorageService.java:1084) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:1032) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.StorageService.initServer(StorageService.java:755) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.StorageService.initServer(StorageService.java:620) > ~[apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:333) > [apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:551) > [apache-cassandra-3.0.6.jar:3.0.6] > at > org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:679) > [apache-cassandra-3.0.6.jar:3.0.6] > {noformat} > A quick glance at code suggests that this race is possible: > {noformat} > In Keyspace.java > public ColumnFamilyStore getColumnFamilyStore(String cfName) > { > UUID id = Schema.instance.getId(getName(), cfName); > if (id == null) > throw new IllegalArgumentException(String.format("Unknown > keyspace/cf pair (%s.%s)", getName(), cfName)); > return getColumnFamilyStore(id); > } > public ColumnFamilyStore getColumnFamilyStore(UUID id) > { > ColumnFamilyStore cfs = columnFamilyStores.get(id); > if (cfs == null) > throw new IllegalArgumentException("Unknown CF " + id); > return cfs; > } > {noformat} > {noformat} > In Schema.java: > public void addTable(CFMetaData cfm) > { > assert getCFMetaData(cfm.ksName, cfm.cfName) == null; > // Make sure the keyspace is initialized > Keyspace.open(cfm.ksName); > // Update the keyspaces map with the updated metadata > update(cfm.ksName, ks -> ks.withSwapped(ks.tables.with(cfm))); > // Update the table ID <-> table name map (cfIdMap) > load(cfm); > // init the new CF before switching the KSM to the new one > // to avoid races as in CASSANDRA-10761 > Keyspace.open(cfm.ksName).initCf(cfm, true); > MigrationManager.instance.notifyCreateColumnFamily(cfm); > } > public void load(CFMetaData cfm) > { > Pair<String, String> key = Pair.create(cfm.ksName, cfm.cfName); > if (cfIdMap.containsKey(key)) > throw new RuntimeException(String.format("Attempting to load > already loaded table %s.%s", cfm.ksName, cfm.cfName)); > logger.debug("Adding {} to cfIdMap", cfm); > cfIdMap.put(key, cfm.cfId); > } > {noformat} > So something can be added to the cfIdMap and not be present in the > columnFamilyStores (which is added in Keyspace.open(cfm.ksName).initCf(cfm, > true); step). -- This message was sent by Atlassian JIRA (v6.3.4#6332)