This likely has to do with hbase scanners running into lease expiration. Try overriding the value of hbase.client.scanner.timeout.period in the server side hbase-site.xml to a large value.
We have a feature coming out in Phoenix 4.7 (soon to be released) that will take care of automatically renewing scanner leases for such long running queries. To take advantage of that feature, you would need to upgrade your HBase version to a version as recent as 0.98.17 if you are using HBase-0.98 or certain 1.x versions (1.3.0 <https://issues.apache.org/jira/browse/HBASE/fixforversion/12332794>, 1.2.1 <https://issues.apache.org/jira/browse/HBASE/fixforversion/12332824>, 1.0.3 <https://issues.apache.org/jira/browse/HBASE/fixforversion/12332968>, 1.1.3 <https://issues.apache.org/jira/browse/HBASE/fixforversion/12333152>) if you are using HBase-1.x. On Tue, Jan 19, 2016 at 2:13 PM, Nick Dimiduk <ndimi...@gmail.com> wrote: > What version of HBase are you using with Phoenix? Does your query ever > finish -- is it log noise or does it lead to real timeouts? > > On Mon, Sep 28, 2015 at 12:35 PM, Konstantinos Kougios < > kostas.koug...@googlemail.com> wrote: > >> I've got a 500 mil rows table on a fairly mediocre cluster. I had some >> timeout issues running a count(*) but those are sorted. Now I get a diff >> issue, any ideas? >> >> 15/09/28 20:24:07 WARN client.ScannerCallable: Ignore, probably already >> closed >> org.apache.hadoop.hbase.UnknownScannerException: >> org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed? >> at >> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223) >> at >> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205) >> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114) >> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101) >> at >> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) >> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) >> at java.lang.Thread.run(Thread.java:745) >> >> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native >> Method) >> at >> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) >> at >> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) >> at java.lang.reflect.Constructor.newInstance(Constructor.java:422) >> at >> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106) >> at >> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95) >> at >> org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:322) >> at >> org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:357) >> at >> org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:195) >> at >> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:142) >> at >> org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:61) >> at >> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) >> at >> org.apache.hadoop.hbase.client.StatsTrackingRpcRetryingCaller.callWithoutRetries(StatsTrackingRpcRetryingCaller.java:56) >> at >> org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320) >> at >> org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:258) >> at >> org.apache.hadoop.hbase.client.ClientScanner.possiblyNextScanner(ClientScanner.java:241) >> at >> org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:532) >> at >> org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:364) >> at >> org.apache.phoenix.iterate.ScanningResultIterator.next(ScanningResultIterator.java:55) >> at >> org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:107) >> at >> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:125) >> at >> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:83) >> at >> org.apache.phoenix.iterate.SpoolingResultIterator.<init>(SpoolingResultIterator.java:62) >> at >> org.apache.phoenix.iterate.SpoolingResultIterator$SpoolingResultIteratorFactory.newIterator(SpoolingResultIterator.java:78) >> at >> org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:109) >> at >> org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:100) >> at java.util.concurrent.FutureTask.run(FutureTask.java:266) >> at >> org.apache.phoenix.job.JobManager$InstrumentedJobFutureTask.run(JobManager.java:183) >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >> at java.lang.Thread.run(Thread.java:745) >> Caused by: >> org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.UnknownScannerException): >> org.apache.hadoop.hbase.UnknownScannerException: Name: 874, already closed? >> at >> org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2223) >> at >> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205) >> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2114) >> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:101) >> at >> org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) >> at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) >> at java.lang.Thread.run(Thread.java:745) >> >> at >> org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1196) >> at >> org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:213) >> at >> org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:287) >> at >> org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32651) >> at >> org.apache.hadoop.hbase.client.ScannerCallable.close(ScannerCallable.java:355) >> ... 23 more >> >> >
