[ https://issues.apache.org/jira/browse/CASSANDRA-15055?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16796144#comment-16796144 ]
Per Otterström commented on CASSANDRA-15055: -------------------------------------------- I believe you get this because the node(s) where permission data is store is temporarily unresponsive. * What's the RF on your system_auth keyspace? * Is this a multi DC setup? Seem related to CASSANDRA-15041 > ReadTimeoutException: Operation timed out - received only 0 responses > --------------------------------------------------------------------- > > Key: CASSANDRA-15055 > URL: https://issues.apache.org/jira/browse/CASSANDRA-15055 > Project: Cassandra > Issue Type: Bug > Reporter: GANGADHARA > Priority: Normal > > we have Apache Cassandra 2.2.5 cluster with 05 nodes running on Ubuntu 8 > core 64 gb memory server , in the system.log we are consistently seeing > ReadTimeoutException, need help on understanding what could be the issue > bombarding the log and fix the issue > > uname -a > Linux subcass004A 4.4.0-128-generic #154~14.04.1-Ubuntu SMP Fri May 25 > 14:58:51 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux > > *free -g* > *total used free shared buffers cached* > *Mem: 55 54 0 0 1 16* > *-/+ buffers/cache: 36 18* > *Swap: 0 0 0* > cat /proc/cpuinfo |grep -i process |wc -l > 8 > *cat /etc/cassandra/cassandra-env.sh |grep -i MAX_HEAP_SIZE* > *MAX_HEAP_SIZE="32G"* > > > > ERROR [SharedPool-Worker-72] 2019-03-12 04:52:42,968 QueryMessage.java:136 - > Unexpected error during query > com.google.common.util.concurrent.UncheckedExecutionException: > *java.lang.RuntimeException: > org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - > received only 0 responses.* > at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2201) > ~[guava-16.0.jar:na] > at com.google.common.cache.LocalCache.get(LocalCache.java:3934) > ~[guava-16.0.jar:na] > at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3938) > ~[guava-16.0.jar:na] > at > com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4821) > ~[guava-16.0.jar:na] > at > org.apache.cassandra.auth.PermissionsCache.getPermissions(PermissionsCache.java:72) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.AuthenticatedUser.getPermissions(AuthenticatedUser.java:104) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.authorize(ClientState.java:367) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.checkPermissionOnResourceChain(ClientState.java:300) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.ensureHasPermission(ClientState.java:277) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:264) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:248) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.statements.SelectStatement.checkAccess(SelectStatement.java:153) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:223) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:257) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:242) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:123) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507) > [apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401) > [apache-cassandra-2.2.5.jar:2.2.5] > at > io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > [na:1.8.0_181] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164) > [apache-cassandra-2.2.5.jar:2.2.5] > > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) > [apache-cassandra-2.2.5.jar:2.2.5] > at java.lang.Thread.run(Thread.java:748) [na:1.8.0_181] > Caused by: java.lang.RuntimeException: > org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - > received only 0 responses. > at > org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:497) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.CassandraRoleManager.isSuper(CassandraRoleManager.java:301) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at org.apache.cassandra.auth.Roles.hasSuperuserStatus(Roles.java:52) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.AuthenticatedUser.isSuper(AuthenticatedUser.java:71) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.CassandraAuthorizer.authorize(CassandraAuthorizer.java:76) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:124) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:121) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3524) > ~[guava-16.0.jar:na] > at > com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2317) > ~[guava-16.0.jar:na] > at > com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2280) > ~[guava-16.0.jar:na] > at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2195) > ~[guava-16.0.jar:na] > ... 25 common frames omitted > Caused by: org.apache.cassandra.exceptions.ReadTimeoutException: Operation > timed out - received only 0 responses. > at org.apache.cassandra.service.ReadCallback.get(ReadCallback.java:110) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.AbstractReadExecutor.get(AbstractReadExecutor.java:147) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1436) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1360) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1275) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:220) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:176) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.CassandraRoleManager.getRoleFromTable(CassandraRoleManager.java:504) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:491) > ~[apache-cassandra-2.2.5.jar:2.2.5] > ... 35 common frames omitted > ERROR [SharedPool-Worker-72] 2019-03-12 04:52:42,969 ErrorMessage.java:336 - > Unexpected exception during request > com.google.common.util.concurrent.UncheckedExecutionException: > java.lang.RuntimeException: > org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - > received only 0 responses. > at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2201) > ~[guava-16.0.jar:na] > at com.google.common.cache.LocalCache.get(LocalCache.java:3934) > ~[guava-16.0.jar:na] > at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3938) > ~[guava-16.0.jar:na] > at > com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4821) > ~[guava-16.0.jar:na] > at > org.apache.cassandra.auth.PermissionsCache.getPermissions(PermissionsCache.java:72) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.AuthenticatedUser.getPermissions(AuthenticatedUser.java:104) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.authorize(ClientState.java:367) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.checkPermissionOnResourceChain(ClientState.java:300) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.ensureHasPermission(ClientState.java:277) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:264) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:248) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at org.apache.cassandra.service.ClientState.hasAccess(ClientState.java:264) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.service.ClientState.hasColumnFamilyAccess(ClientState.java:248) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.statements.SelectStatement.checkAccess(SelectStatement.java:153) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:223) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:257) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:242) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.transport.messages.QueryMessage.execute(QueryMessage.java:123) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507) > [apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401) > [apache-cassandra-2.2.5.jar:2.2.5] > at > io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:333) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext.access$700(AbstractChannelHandlerContext.java:32) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > io.netty.channel.AbstractChannelHandlerContext$8.run(AbstractChannelHandlerContext.java:324) > [netty-all-4.0.23.Final.jar:4.0.23.Final] > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > [na:1.8.0_181] > at > org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164) > [apache-cassandra-2.2.5.jar:2.2.5] > at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) > [apache-cassandra-2.2.5.jar:2.2.5] > at java.lang.Thread.run(Thread.java:748) [na:1.8.0_181] > Caused by: java.lang.RuntimeException: > org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - > received only 0 responses. > at > org.apache.cassandra.auth.CassandraRoleManager.getRole(CassandraRoleManager.java:497) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.CassandraRoleManager.isSuper(CassandraRoleManager.java:301) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at org.apache.cassandra.auth.Roles.hasSuperuserStatus(Roles.java:52) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.AuthenticatedUser.isSuper(AuthenticatedUser.java:71) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.CassandraAuthorizer.authorize(CassandraAuthorizer.java:76) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:124) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > org.apache.cassandra.auth.PermissionsCache$1.load(PermissionsCache.java:121) > ~[apache-cassandra-2.2.5.jar:2.2.5] > at > com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3524) > ~[guava-16.0.jar:na] > at > com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2317) > ~[guava-16.0.jar:na] > at > com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2280) > ~[guava-16.0.jar:na] > at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2195) > ~[guava-16.0.jar:na] > ... 25 common frames omitted -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org