[
https://issues.apache.org/jira/browse/CASSANDRA-15055?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16796179#comment-16796179
]
GANGADHARA commented on CASSANDRA-15055:
----------------------------------------
Below is our key space detail, it is not multi DC set up , we are not
seeing any consistency errors/warnings relates to any of the tables of
keyspace system_auth in the system.log
Caused by: org.apache.cassandra.exceptions.UnavailableException: Cannot achieve
consistency level QUORUM ---> *no this kind errors seen on any our C*
node's system.log*
desc system_auth
CREATE KEYSPACE system_auth WITH replication = \{'class':
'NetworkTopologyStrategy', 'uswest': '3'} AND durable_writes = true;
CREATE TABLE system_auth.resource_role_permissons_index (
resource text,
role text,
PRIMARY KEY (resource, role)
) WITH CLUSTERING ORDER BY (role ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = '\{"keys":"ALL", "rows_per_partition":"NONE"}'
AND comment = 'index of db roles with permissions granted on a resource'
AND compaction = \{'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'}
AND compression = \{'sstable_compression':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND dclocal_read_repair_chance = 0.0
AND default_time_to_live = 0
AND gc_grace_seconds = 7776000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 3600000
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
CREATE TABLE system_auth.role_permissions (
role text,
resource text,
permissions set<text>,
PRIMARY KEY (role, resource)
) WITH CLUSTERING ORDER BY (resource ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = '\{"keys":"ALL", "rows_per_partition":"NONE"}'
AND comment = 'permissions granted to db roles'
AND compaction = \{'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'}
AND compression = \{'sstable_compression':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND dclocal_read_repair_chance = 0.0
AND default_time_to_live = 0
AND gc_grace_seconds = 7776000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 3600000
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
CREATE TABLE system_auth.role_members (
role text,
member text,
PRIMARY KEY (role, member)
) WITH CLUSTERING ORDER BY (member ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = '\{"keys":"ALL", "rows_per_partition":"NONE"}'
AND comment = 'role memberships lookup table'
AND compaction = \{'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'}
AND compression = \{'sstable_compression':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND dclocal_read_repair_chance = 0.0
AND default_time_to_live = 0
AND gc_grace_seconds = 7776000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 3600000
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
CREATE TABLE system_auth.roles (
role text PRIMARY KEY,
can_login boolean,
is_superuser boolean,
member_of set<text>,
salted_hash text
) WITH bloom_filter_fp_chance = 0.01
AND caching = '\{"keys":"ALL", "rows_per_partition":"NONE"}'
AND comment = 'role definitions'
AND compaction = \{'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy'}
AND compression = \{'sstable_compression':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND dclocal_read_repair_chance = 0.0
AND default_time_to_live = 0
AND gc_grace_seconds = 7776000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 3600000
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
> ReadTimeoutException: Operation timed out - received only 0 responses
> ---------------------------------------------------------------------
>
> Key: CASSANDRA-15055
> URL: https://issues.apache.org/jira/browse/CASSANDRA-15055
> Project: Cassandra
> Issue Type: Bug
> Reporter: GANGADHARA
> Priority: Normal
>
> we have Apache Cassandra 2.2.5 cluster with 05 nodes running on Ubuntu 8
> core 64 gb memory server , in the system.log we are consistently seeing
> ReadTimeoutException, need help on understanding what could be the issue
> bombarding the log and fix the issue
>
> uname -a
> Linux subcass004A 4.4.0-128-generic #154~14.04.1-Ubuntu SMP Fri May 25
> 14:58:51 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
>
> *free -g*
> *total used free shared buffers cached*
> *Mem: 55 54 0 0 1 16*
> *-/+ buffers/cache: 36 18*
> *Swap: 0 0 0*
> cat /proc/cpuinfo |grep -i process |wc -l
> 8
> *cat /etc/cassandra/cassandra-env.sh |grep -i MAX_HEAP_SIZE*
> *MAX_HEAP_SIZE="32G"*
>
>
>
> ERROR [SharedPool-Worker-72] 2019-03-12 04:52:42,968 QueryMessage.java:136 -
> Unexpected error during query
> com.google.common.util.concurrent.UncheckedExecutionException:
> *java.lang.RuntimeException:
> org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out -
> received only 0 responses.*
> at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2201)
> ~[guava-16.0.jar:na]
> at com.google.common.cache.LocalCache.get(LocalCache.java:3934)
> ~[guava-16.0.jar:na]
> at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3938)
> ~[guava-16.0.jar:na]
> at
> com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4821)
> ~[guava-16.0.jar:na]
> at
> org.apache.cassandra.auth.PermissionsCache.getPermissions(PermissionsCache.java:72)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.AuthenticatedUser.getPermissions(AuthenticatedUser.java:104)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.authorize(ClientState.java:367)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.checkPermissionOnResourceChain(ClientState.java:300)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.ensureHasPermission(ClientState.java:277)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:264)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:248)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.statements.SelectStatement.checkAccess(SelectStatement.java:153)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:223)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:257)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:242)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:123)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at
> io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [na:1.8.0_181]
> at
> org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164)
> [apache-cassandra-2.2.5.jar:2.2.5]
>
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at java.lang.Thread.run(Thread.java:748) [na:1.8.0_181]
> Caused by: java.lang.RuntimeException:
> org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out -
> received only 0 responses.
> at
> org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:497)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.CassandraRoleManager.isSuper(CassandraRoleManager.java:301)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at org.apache.cassandra.auth.Roles.hasSuperuserStatus(Roles.java:52)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.AuthenticatedUser.isSuper(AuthenticatedUser.java:71)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.CassandraAuthorizer.authorize(CassandraAuthorizer.java:76)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:124)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:121)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3524)
> ~[guava-16.0.jar:na]
> at
> com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2317)
> ~[guava-16.0.jar:na]
> at
> com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2280)
> ~[guava-16.0.jar:na]
> at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2195)
> ~[guava-16.0.jar:na]
> ... 25 common frames omitted
> Caused by: org.apache.cassandra.exceptions.ReadTimeoutException: Operation
> timed out - received only 0 responses.
> at org.apache.cassandra.service.ReadCallback.get(ReadCallback.java:110)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.AbstractReadExecutor.get(AbstractReadExecutor.java:147)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1436)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1360)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1275)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:220)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:176)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.CassandraRoleManager.getRoleFromTable(CassandraRoleManager.java:504)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:491)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> ... 35 common frames omitted
> ERROR [SharedPool-Worker-72] 2019-03-12 04:52:42,969 ErrorMessage.java:336 -
> Unexpected exception during request
> com.google.common.util.concurrent.UncheckedExecutionException:
> java.lang.RuntimeException:
> org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out -
> received only 0 responses.
> at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2201)
> ~[guava-16.0.jar:na]
> at com.google.common.cache.LocalCache.get(LocalCache.java:3934)
> ~[guava-16.0.jar:na]
> at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3938)
> ~[guava-16.0.jar:na]
> at
> com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4821)
> ~[guava-16.0.jar:na]
> at
> org.apache.cassandra.auth.PermissionsCache.getPermissions(PermissionsCache.java:72)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.AuthenticatedUser.getPermissions(AuthenticatedUser.java:104)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.authorize(ClientState.java:367)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.checkPermissionOnResourceChain(ClientState.java:300)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.ensureHasPermission(ClientState.java:277)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:264)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:248)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:264)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:248)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.statements.SelectStatement.checkAccess(SelectStatement.java:153)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:223)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:257)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:242)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:123)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at
> io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324)
> [netty-all-4.0.23.Final.jar:4.0.23.Final]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> [na:1.8.0_181]
> at
> org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105)
> [apache-cassandra-2.2.5.jar:2.2.5]
> at java.lang.Thread.run(Thread.java:748) [na:1.8.0_181]
> Caused by: java.lang.RuntimeException:
> org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out -
> received only 0 responses.
> at
> org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:497)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.CassandraRoleManager.isSuper(CassandraRoleManager.java:301)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at org.apache.cassandra.auth.Roles.hasSuperuserStatus(Roles.java:52)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.AuthenticatedUser.isSuper(AuthenticatedUser.java:71)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.CassandraAuthorizer.authorize(CassandraAuthorizer.java:76)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:124)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:121)
> ~[apache-cassandra-2.2.5.jar:2.2.5]
> at
> com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3524)
> ~[guava-16.0.jar:na]
> at
> com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2317)
> ~[guava-16.0.jar:na]
> at
> com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2280)
> ~[guava-16.0.jar:na]
> at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2195)
> ~[guava-16.0.jar:na]
> ... 25 common frames omitted
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
