HI Team,
We have upgraded the phoenix from 4.7.0 to 4.11.0 and started noticing the
attached exception.
Can you help me identifying the root cause of the exception ? Thanks.
Regards,
Anil
2018-03-21 08:13:19,684 ERROR
com.tst.hadoop.flume.writer.inventory.AccountPersistenceImpl: Error querying
account UUIDs for siteId 0101293035
org.springframework.dao.DataAccessResourceFailureException:
PreparedStatementCallback; SQL [select uuids from account where siteid = ?];
java.lang.RuntimeException: java.util.concurrent.RejectedExecutionException:
Task
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@5ddc0ec5
rejected from java.util.concurrent.ThreadPoolExecutor@5204afb0[Shutting down,
pool size = 59, active threads = 0, queued tasks = 0, completed tasks =
2667625]; nested exception is org.apache.phoenix.exception.PhoenixIOException:
java.lang.RuntimeException: java.util.concurrent.RejectedExecutionException:
Task
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@5ddc0ec5
rejected from java.util.concurrent.ThreadPoolExecutor@5204afb0[Shutting down,
pool size = 59, active threads = 0, queued tasks = 0, completed tasks = 2667625]
at
org.springframework.jdbc.support.SQLStateSQLExceptionTranslator.doTranslate(SQLStateSQLExceptionTranslator.java:105)
at
org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:73)
at
org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
at
org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
at
org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:660)
at
org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:695)
at
org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:722)
at
org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:772)
at
org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate.query(NamedParameterJdbcTemplate.java:192)
at
org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate.queryForList(NamedParameterJdbcTemplate.java:290)
at
com.tst.hadoop.flume.writer.inventory.AccountPersistenceImpl.getUUIDs(AccountPersistenceImpl.java:187)
at
net.juniper.spark.stream.sap.data.mapper.ServiceContractMapper.populatePartnerAccountId(ServiceContractMapper.java:134)
at
net.juniper.spark.stream.sap.data.mapper.ServiceContractMapper.map(ServiceContractMapper.java:70)
at
net.juniper.spark.stream.sap.data.mapper.ServiceContractMapper.map(ServiceContractMapper.java:39)
at
net.juniper.spark.stream.sap.processor.SAPDataProcessor.mapObject(SAPDataProcessor.java:34)
at
net.juniper.spark.stream.sap.processor.SAPDataProcessor.processData(SAPDataProcessor.java:43)
at
com.tst.hadoop.flume.sink.SAPContractCustomSink.process(SAPContractCustomSink.java:113)
at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:67)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.phoenix.exception.PhoenixIOException:
java.lang.RuntimeException: java.util.concurrent.RejectedExecutionException:
Task
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@5ddc0ec5
rejected from java.util.concurrent.ThreadPoolExecutor@5204afb0[Shutting down,
pool size = 59, active threads = 0, queued tasks = 0, completed tasks = 2667625]
at
org.apache.phoenix.util.ServerUtil.parseServerException(ServerUtil.java:116)
at
org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:875)
at
org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:819)
at
org.apache.phoenix.iterate.RoundRobinResultIterator.getIterators(RoundRobinResultIterator.java:176)
at
org.apache.phoenix.iterate.RoundRobinResultIterator.next(RoundRobinResultIterator.java:91)
at
org.apache.phoenix.jdbc.PhoenixResultSet.next(PhoenixResultSet.java:778)
at
org.apache.commons.dbcp.DelegatingResultSet.next(DelegatingResultSet.java:207)
at
org.apache.commons.dbcp.DelegatingResultSet.next(DelegatingResultSet.java:207)
at
org.springframework.jdbc.core.RowMapperResultSetExtractor.extractData(RowMapperResultSetExtractor.java:92)
at
org.springframework.jdbc.core.RowMapperResultSetExtractor.extractData(RowMapperResultSetExtractor.java:60)
at
org.springframework.jdbc.core.JdbcTemplate$1.doInPreparedStatement(JdbcTemplate.java:708)
at
org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:644)
... 15 more
Caused by: java.util.concurrent.ExecutionException: java.lang.RuntimeException:
java.util.concurrent.RejectedExecutionException: Task
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@5ddc0ec5
rejected from java.util.concurrent.ThreadPoolExecutor@5204afb0[Shutting down,
pool size = 59, active threads = 0, queued tasks = 0, completed tasks = 2667625]
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:202)
at
org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:870)
... 25 more
Caused by: java.lang.RuntimeException:
java.util.concurrent.RejectedExecutionException: Task
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@5ddc0ec5
rejected from java.util.concurrent.ThreadPoolExecutor@5204afb0[Shutting down,
pool size = 59, active threads = 0, queued tasks = 0, completed tasks = 2667625]
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:218)
at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:327)
at
org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:410)
at
org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:371)
at
org.apache.phoenix.iterate.ScanningResultIterator.next(ScanningResultIterator.java:96)
at
org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:140)
at
org.apache.phoenix.iterate.LookAheadResultIterator$1.advance(LookAheadResultIterator.java:47)
at
org.apache.phoenix.iterate.LookAheadResultIterator.init(LookAheadResultIterator.java:59)
at
org.apache.phoenix.iterate.LookAheadResultIterator.peek(LookAheadResultIterator.java:73)
at
org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:126)
at
org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:111)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at
org.apache.phoenix.job.JobManager$InstrumentedJobFutureTask.run(JobManager.java:183)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
... 1 more
Caused by: java.util.concurrent.RejectedExecutionException: Task
org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@5ddc0ec5
rejected from java.util.concurrent.ThreadPoolExecutor@5204afb0[Shutting down,
pool size = 59, active threads = 0, queued tasks = 0, completed tasks = 2667625]
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
at
org.apache.hadoop.hbase.client.ResultBoundedCompletionService.submit(ResultBoundedCompletionService.java:171)
at
org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.addCallsForCurrentReplica(ScannerCallableWithReplicas.java:306)
at
org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:171)
at
org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:61)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:210)
... 15 more
2018-03-21 08:13:19,690 ERROR
com.tst.hadoop.flume.writer.inventory.AccountPersistenceImpl: Error persisting
account for siteId 0101210927
org.springframework.jdbc.UncategorizedSQLException: PreparedStatementCallback;
uncategorized SQLException for SQL [upsert into account (siteid, uuids,
accName, pcvdAccName, pcvdAccId, accId) values (?,?, ?, ?, ?, ?)]; SQL state
[null]; error code [0]; java.lang.IllegalArgumentException: Connection is null
or closed.; nested exception is java.sql.SQLException:
java.lang.IllegalArgumentException: Connection is null or closed.
at
org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:84)
at
org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
at
org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
at
org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:660)
at
org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:909)
at
org.springframework.jdbc.core.JdbcTemplate.update(JdbcTemplate.java:933)
at
org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate.update(NamedParameterJdbcTemplate.java:313)
at
com.tst.hadoop.flume.writer.inventory.AccountPersistenceImpl.persist(AccountPersistenceImpl.java:126)
at
com.tst.hadoop.flume.writer.inventory.AccountPersistenceImpl.createEmptyAccount(AccountPersistenceImpl.java:794)
at
net.juniper.spark.stream.sap.data.mapper.ServiceContractMapper.populatePartnerAccountId(ServiceContractMapper.java:139)
at
net.juniper.spark.stream.sap.data.mapper.ServiceContractMapper.map(ServiceContractMapper.java:70)
at
net.juniper.spark.stream.sap.data.mapper.ServiceContractMapper.map(ServiceContractMapper.java:39)
at
net.juniper.spark.stream.sap.processor.SAPDataProcessor.mapObject(SAPDataProcessor.java:34)
at
net.juniper.spark.stream.sap.processor.SAPDataProcessor.processData(SAPDataProcessor.java:43)
at
com.tst.hadoop.flume.sink.SAPContractCustomSink.process(SAPContractCustomSink.java:113)
at
org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:67)
at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:145)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.sql.SQLException: java.lang.IllegalArgumentException:
Connection is null or closed.
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.metaDataCoprocessorExec(ConnectionQueryServicesImpl.java:1331)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.metaDataCoprocessorExec(ConnectionQueryServicesImpl.java:1293)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.getTable(ConnectionQueryServicesImpl.java:1510)
at
org.apache.phoenix.schema.MetaDataClient.updateCache(MetaDataClient.java:596)
at
org.apache.phoenix.schema.MetaDataClient.updateCache(MetaDataClient.java:519)
at
org.apache.phoenix.schema.MetaDataClient.updateCache(MetaDataClient.java:511)
at
org.apache.phoenix.schema.MetaDataClient.updateCache(MetaDataClient.java:507)
at
org.apache.phoenix.execute.MutationState.validate(MutationState.java:805)
at
org.apache.phoenix.execute.MutationState.validateAll(MutationState.java:793)
at
org.apache.phoenix.execute.MutationState.send(MutationState.java:1005)
at
org.apache.phoenix.execute.MutationState.send(MutationState.java:1496)
at
org.apache.phoenix.execute.MutationState.commit(MutationState.java:1328)
at
org.apache.phoenix.jdbc.PhoenixConnection$3.call(PhoenixConnection.java:562)
at
org.apache.phoenix.jdbc.PhoenixConnection$3.call(PhoenixConnection.java:559)
at org.apache.phoenix.jdbc.PhoenixConnection$3.call(PhoenixConnection.java:559)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
at
org.apache.phoenix.jdbc.PhoenixConnection.commit(PhoenixConnection.java:559)
at
org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:396)
at
org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
at
org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:375)
at
org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
at
org.apache.phoenix.jdbc.PhoenixPreparedStatement.executeUpdate(PhoenixPreparedStatement.java:199)
at
org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at
org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at
org.springframework.jdbc.core.JdbcTemplate$2.doInPreparedStatement(JdbcTemplate.java:916)
at
org.springframework.jdbc.core.JdbcTemplate$2.doInPreparedStatement(JdbcTemplate.java:909)
at
org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:644)
... 14 more
Caused by: java.lang.IllegalArgumentException: Connection is null or closed.
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:373)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getTable(ConnectionManager.java:761)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getTable(ConnectionManager.java:743)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getTable(ConnectionManager.java:738)
at
org.apache.phoenix.query.HTableFactory$HTableFactoryImpl.getTable(HTableFactory.java:51)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.getTable(ConnectionQueryServicesImpl.java:462)
at
org.apache.phoenix.query.ConnectionQueryServicesImpl.metaDataCoprocessorExec(ConnectionQueryServicesImpl.java:1310)
... 40 more
