[ https://issues.apache.org/jira/browse/PHOENIX-3164?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15421947#comment-15421947 ]
Hudson commented on PHOENIX-3164: --------------------------------- SUCCESS: Integrated in Jenkins build Phoenix-master #1361 (See [https://builds.apache.org/job/Phoenix-master/1361/]) PHOENIX-3164 Cache UGI instances for remote users in PQS (elserj: rev 44cf0827f1e5577510ee95f149df8d552a56da3c) * (edit) phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java * (add) phoenix-queryserver/src/test/java/org/apache/phoenix/queryserver/server/PhoenixDoAsCallbackTest.java * (edit) phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java * (edit) phoenix-queryserver/src/main/java/org/apache/phoenix/queryserver/server/Main.java > PhoenixConnection leak in PQS with security enabled > --------------------------------------------------- > > Key: PHOENIX-3164 > URL: https://issues.apache.org/jira/browse/PHOENIX-3164 > Project: Phoenix > Issue Type: Bug > Affects Versions: 4.8.0 > Reporter: Josh Elser > Assignee: Josh Elser > Priority: Critical > Fix For: 4.9.0, 4.8.1 > > Attachments: PHOENIX-3164.001.patch, PHOENIX-3164.002.patch > > > Noticed this one yesterday in some testing. PQS clients were getting stuck in > a loop trying to find the location of the hbase:meta region, but never > actually finding it despite HBase appearing to be 100% healthy. > In PQS: > {noformat} > "qtp1908490900-20" daemon prio=10 tid=0x00007f67284ae800 nid=0x72b8 waiting > on condition [0x00007f66f570a000] > java.lang.Thread.State: TIMED_WAITING (sleeping) > at java.lang.Thread.sleep(Native Method) > at java.lang.Thread.sleep(Thread.java:340) > at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:360) > at > org.apache.hadoop.hbase.util.RetryCounter.sleepUntilNextRetry(RetryCounter.java:158) > at > org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:373) > at org.apache.hadoop.hbase.zookeeper.ZKUtil.getData(ZKUtil.java:622) > at > org.apache.hadoop.hbase.zookeeper.MetaTableLocator.getMetaRegionState(MetaTableLocator.java:491) > at > org.apache.hadoop.hbase.zookeeper.MetaTableLocator.getMetaRegionLocation(MetaTableLocator.java:172) > at > org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:608) > at > org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:589) > at > org.apache.hadoop.hbase.zookeeper.MetaTableLocator.blockUntilAvailable(MetaTableLocator.java:568) > at > org.apache.hadoop.hbase.client.ZooKeeperRegistry.getMetaRegionLocation(ZooKeeperRegistry.java:61) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateMeta(ConnectionManager.java:1192) > - locked <0x000000070b109930> (a java.lang.Object) > at > org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1159) > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:300) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:156) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) > at > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) > at > org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:326) > at > org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:301) > at > org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:166) > at > org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:161) > at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:794) > at > org.apache.hadoop.hbase.MetaTableAccessor.fullScan(MetaTableAccessor.java:602) > at > org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:366) > at > org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:405) > at > org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2358) > - locked <0x000000070b0c4d58> (a > org.apache.phoenix.query.ConnectionQueryServicesImpl) > at > org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2327) > at > org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:78) > at > org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2327) > at > org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:233) > at > org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:144) > at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:202) > at java.sql.DriverManager.getConnection(DriverManager.java:571) > at java.sql.DriverManager.getConnection(DriverManager.java:187) > at > org.apache.calcite.avatica.jdbc.JdbcMeta.openConnection(JdbcMeta.java:616) > - locked <0x000000070538c5e0> (a > org.apache.calcite.avatica.jdbc.JdbcMeta) > at > org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:299) > at > org.apache.calcite.avatica.remote.Service$OpenConnectionRequest.accept(Service.java:1748) > at > org.apache.calcite.avatica.remote.Service$OpenConnectionRequest.accept(Service.java:1728) > at > org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:95) > at > org.apache.calcite.avatica.remote.ProtobufHandler.apply(ProtobufHandler.java:46) > at > org.apache.calcite.avatica.server.AvaticaProtobufHandler$2.call(AvaticaProtobufHandler.java:120) > at > org.apache.calcite.avatica.server.AvaticaProtobufHandler$2.call(AvaticaProtobufHandler.java:118) > at > org.apache.phoenix.queryserver.server.Main$PhoenixDoAsCallback$1.run(Main.java:282) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:415) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724) > at > org.apache.phoenix.queryserver.server.Main$PhoenixDoAsCallback.doAsRemoteUser(Main.java:279) > at > org.apache.calcite.avatica.server.HttpServer$Builder$1.doAsRemoteUser(HttpServer.java:648) > at > org.apache.calcite.avatica.server.AvaticaProtobufHandler.handle(AvaticaProtobufHandler.java:117) > at > org.apache.phoenix.shaded.org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:542) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.Server.handle(Server.java:499) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) > at > org.apache.phoenix.shaded.org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) > at > org.apache.phoenix.shaded.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) > at > org.apache.phoenix.shaded.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) > at java.lang.Thread.run(Thread.java:745) > {noformat} > And on the clients: > {noformat} > AvaticaClientRuntimeException: Remote driver error: RuntimeException: > java.sql.SQLException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > locations -> SQLException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > locations -> RetriesExhaustedException: Can't get the locations. Error -1 > (00000) null > java.lang.RuntimeException: java.sql.SQLException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > locations > at org.apache.calcite.avatica.jdbc.JdbcMeta.openConnection(JdbcMeta.java:619) > at org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:299) > at > org.apache.calcite.avatica.remote.Service$OpenConnectionRequest.accept(Service.java:1748) > at > org.apache.calcite.avatica.remote.Service$OpenConnectionRequest.accept(Service.java:1728) > at > org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:95) > at > org.apache.calcite.avatica.remote.ProtobufHandler.apply(ProtobufHandler.java:46) > at > org.apache.calcite.avatica.server.AvaticaProtobufHandler$2.call(AvaticaProtobufHandler.java:120) > at > org.apache.calcite.avatica.server.AvaticaProtobufHandler$2.call(AvaticaProtobufHandler.java:118) > at > org.apache.phoenix.queryserver.server.Main$PhoenixDoAsCallback$1.run(Main.java:282) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:415) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724) > at > org.apache.phoenix.queryserver.server.Main$PhoenixDoAsCallback.doAsRemoteUser(Main.java:279) > at > org.apache.calcite.avatica.server.HttpServer$Builder$1.doAsRemoteUser(HttpServer.java:648) > at > org.apache.calcite.avatica.server.AvaticaProtobufHandler.handle(AvaticaProtobufHandler.java:117) > at > org.apache.phoenix.shaded.org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:542) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.Server.handle(Server.java:499) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) > at > org.apache.phoenix.shaded.org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) > at > org.apache.phoenix.shaded.org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) > at > org.apache.phoenix.shaded.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) > at > org.apache.phoenix.shaded.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.sql.SQLException: > org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't get the > locations > at > org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2590) > at > org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2327) > at > org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:78) > at > org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2327) > at > org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:233) > at > org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:144) > at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:202) > at java.sql.DriverManager.getConnection(DriverManager.java:571) > at java.sql.DriverManager.getConnection(DriverManager.java:187) > at org.apache.calcite.avatica.jdbc.JdbcMeta.openConnection(JdbcMeta.java:616) > ... 24 more > Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Can't > get the locations > at > org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.getRegionLocations(RpcRetryingCallerWithReadReplicas.java:312) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:156) > at > org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) > at > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) > at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:326) > at > org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:301) > at > org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:166) > at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:161) > at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:794) > at > org.apache.hadoop.hbase.MetaTableAccessor.fullScan(MetaTableAccessor.java:602) > at > org.apache.hadoop.hbase.MetaTableAccessor.tableExists(MetaTableAccessor.java:366) > at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:405) > at > org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2358) > ... 33 more > at > org.apache.calcite.avatica.remote.Service$ErrorResponse.toException(Service.java:2453) > at > org.apache.calcite.avatica.remote.RemoteProtobufService._apply(RemoteProtobufService.java:61) > at > org.apache.calcite.avatica.remote.ProtobufService.apply(ProtobufService.java:81) > at org.apache.calcite.avatica.remote.Driver.connect(Driver.java:175) > at sqlline.DatabaseConnection.connect(DatabaseConnection.java:157) > at sqlline.DatabaseConnection.getConnection(DatabaseConnection.java:203) > at sqlline.Commands.connect(Commands.java:1064) > at sqlline.Commands.connect(Commands.java:996) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at sqlline.ReflectiveCommandHandler.execute(ReflectiveCommandHandler.java:36) > at sqlline.SqlLine.dispatch(SqlLine.java:804) > at sqlline.SqlLine.initArgs(SqlLine.java:588) > at sqlline.SqlLine.begin(SqlLine.java:656) > at sqlline.SqlLine.start(SqlLine.java:398) > at sqlline.SqlLine.main(SqlLine.java:292) > at > org.apache.phoenix.queryserver.client.SqllineWrapper$1.run(SqllineWrapper.java:78) > at > org.apache.phoenix.queryserver.client.SqllineWrapper$1.run(SqllineWrapper.java:75) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:415) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724) > at > org.apache.phoenix.queryserver.client.SqllineWrapper.main(SqllineWrapper.java:75) > {noformat} > Best as I can tell so far, this is due to PHOENIX-3126 and PQS not caching > UGI instances. hashCode() and equals() on UGI work on references (not on > values). I believe the simple fix is to just add some caching of UGI > instances for the end-users in PQS. -- This message was sent by Atlassian JIRA (v6.3.4#6332)