Hello!
I have a problem with long-running HBase-client. I have the
HBase-0.98.6-cdh5.3.1.
After few days of application run I have a bug (?) with a trace as
below. As I debugged my issue I see, that the client try to reach
region, that does not exists. The region has existed in the past,
hoverer has been recently merged or migrated. The client (?) doesn't
refresh information about a region for a record and because of this
cannot get the existing record. Restart of the client-application
helps. Is it a bug or am I missing something in a configuration?
java.net.SocketTimeoutException: callTimeout=100, callDuration=2307:
row 'abcd' on table 'test:REC-DOMAIN-ARTICLE_ID
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:141)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at org.apache.hadoop.hbase.client.HTable.get(HTable.java:781)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
pl.com.agora.bigdata.recommendation_service.db.hbase.HBaseRecommendationDao.getRecommendations(HBaseRecommendationDao.java:52)
~[HBaseRecommendationDao.class:?]
at
pl.com.agora.bigdata.recommendation_service.logic.RecommendationEngine.fetchRecommendations(RecommendationEngine.java:552)
[RecommendationEngine.class:?]
at
pl.com.agora.bigdata.recommendation_service.logic.RecommendationEngine.getRecommendations(RecommendationEngine.java:177)
[RecommendationEngine.class:?]
at
pl.com.agora.bigdata.recommendation_service.services.RecommendationService.getRecommendation(RecommendationService.java:84)
[RecommendationService.class:?]
at sun.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_25]
at java.lang.reflect.Method.invoke(Method.java:483) ~[?:1.8.0_25]
at
org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221)
[spring-web-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:137)
[spring-web-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:110)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:777)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:706)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:943)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:877)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:966)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at
org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:868)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:644)
[servlet-api.jar:?]
at
org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:842)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:725)
[servlet-api.jar:?]
at
org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:842)
[spring-webmvc-4.1.5.RELEASE.jar:4.1.5.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:725)
[servlet-api.jar:?]
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:291)
[catalina.jar:8.0.20]
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
[catalina.jar:8.0.20]
at
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
[tomcat-websocket.jar:8.0.20]
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:239)
[catalina.jar:8.0.20]
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
[catalina.jar:8.0.20]
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
[catalina.jar:8.0.20]
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106)
[catalina.jar:8.0.20]
at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
[catalina.jar:8.0.20]
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:142)
[catalina.jar:8.0.20]
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
[catalina.jar:8.0.20]
at
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:610)
[catalina.jar:8.0.20]
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
[catalina.jar:8.0.20]
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:516)
[catalina.jar:8.0.20]
at
org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1086)
[tomcat-coyote.jar:8.0.20]
at
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:659)
[tomcat-coyote.jar:8.0.20]
at
org.apache.coyote.http11.Http11NioProtocol$Http11ConnectionHandler.process(Http11NioProtocol.java:223)
[tomcat-coyote.jar:8.0.20]
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1558)
[tomcat-coyote.jar:8.0.20]
at
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1515)
[tomcat-coyote.jar:8.0.20]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0_25]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0_25]
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
[tomcat-util.jar:8.0.20]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_25]
Caused by: org.apache.hadoop.hbase.NotServingRegionException:
org.apache.hadoop.hbase.NotServingRegionException: Region
test:REC-DOMAIN-ARTICLE_ID,,1432022099159.d821b6a1362b39f13227491aeb40717f.
is not online on sqhadoop11.gazeta.pl,60020,1437623492363
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2761)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4256)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2894)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108)
at
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96)
at java.lang.Thread.run(Thread.java:745)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method) ~[?:1.8.0_25]
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
~[?:1.8.0_25]
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
~[?:1.8.0_25]
at java.lang.reflect.Constructor.newInstance(Constructor.java:408)
~[?:1.8.0_25]
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
~[hadoop-common-2.5.0-cdh5.3.1.jar:?]
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
~[hadoop-common-2.5.0-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:304)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.get(ProtobufUtil.java:1526)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:777)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:775)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:114)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
... 43 more
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException:
org.apache.hadoop.hbase.NotServingRegionException: Region
test:REC-DOMAIN-ARTICLE_ID,,1432022099159.d821b6a1362b39f13227491aeb40717f.
is not online on sqhadoop11.gazeta.pl,60020,1437623492363
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2761)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:4256)
at
org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2894)
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108)
at
org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116)
at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96)
at java.lang.Thread.run(Thread.java:745)
at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1457)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1661)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1719)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:30304)
~[hbase-protocol-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.protobuf.ProtobufUtil.get(ProtobufUtil.java:1522)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:777)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:775)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:114)
~[hbase-client-0.98.6-cdh5.3.1.jar:?]
... 43 more
--
Kindly regards/ Pozdrawiam,
Wojciech Indyk
